[dpdk-dev] [PATCH 0/3] Add link status notification support to Mellanox PMDs

[dpdk-dev] [PATCH 0/3] Add link status notification support to Mellanox PMDs

[Adrien Mazarguil]

Mellanox PMDs (mlx4 and mlx5) support link status notification with
libibverbs through a file descriptor, in a manner reminiscent of UIO/VFIO.

It still requires a new "external" interrupt handler type since contrary
to other types, this file descriptor must not be processed by EAL.

Nelio Laranjeiro (3):
  eal: new interrupt handler type
  mlx4: handle interrupts
  mlx5: handle interrupts

 drivers/net/mlx4/mlx4.c                            | 166 +++++++++++++++++++++
 drivers/net/mlx4/mlx4.h                            |   3 +
 drivers/net/mlx5/mlx5.c                            |   3 +
 drivers/net/mlx5/mlx5.h                            |   7 +
 drivers/net/mlx5/mlx5_defs.h                       |   3 +
 drivers/net/mlx5/mlx5_ethdev.c                     | 150 +++++++++++++++++++
 drivers/net/mlx5/mlx5_trigger.c                    |   4 +
 lib/librte_eal/linuxapp/eal/eal_interrupts.c       |  34 +++--
 .../linuxapp/eal/include/exec-env/rte_interrupts.h |   1 +
 9 files changed, 356 insertions(+), 15 deletions(-)

-- 
2.1.0

[dpdk-dev] [PATCH 1/3] eal: new interrupt handler type

[Adrien Mazarguil]

From: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

Add RTE_INTR_HANDLE_EXT handler type for PMDs that do not support VFIO or
UIO.  Those are expected to manage the file descriptor themselves.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 lib/librte_eal/linuxapp/eal/eal_interrupts.c       | 34 ++++++++++++----------
 .../linuxapp/eal/include/exec-env/rte_interrupts.h |  1 +
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
index 078318c..79eedaf 100644
--- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
@@ -691,26 +691,30 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 			bytes_read = sizeof(buf.vfio_intr_count);
 			break;
 #endif
+		case RTE_INTR_HANDLE_EXT:
 		default:
 			bytes_read = 1;
 			break;
 		}
 
-		/**
-		 * read out to clear the ready-to-be-read flag
-		 * for epoll_wait.
-		 */
-		bytes_read = read(events[n].data.fd, &buf, bytes_read);
-		if (bytes_read < 0) {
-			if (errno == EINTR || errno == EWOULDBLOCK)
-				continue;
-
-			RTE_LOG(ERR, EAL, "Error reading from file "
-				"descriptor %d: %s\n", events[n].data.fd,
-							strerror(errno));
-		} else if (bytes_read == 0)
-			RTE_LOG(ERR, EAL, "Read nothing from file "
-				"descriptor %d\n", events[n].data.fd);
+		if (src->intr_handle.type != RTE_INTR_HANDLE_EXT) {
+			/**
+			 * read out to clear the ready-to-be-read flag
+			 * for epoll_wait.
+			 */
+			bytes_read = read(events[n].data.fd, &buf, bytes_read);
+			if (bytes_read < 0) {
+				if (errno == EINTR || errno == EWOULDBLOCK)
+					continue;
+
+				RTE_LOG(ERR, EAL, "Error reading from file "
+					"descriptor %d: %s\n",
+					events[n].data.fd,
+					strerror(errno));
+			} else if (bytes_read == 0)
+				RTE_LOG(ERR, EAL, "Read nothing from file "
+					"descriptor %d\n", events[n].data.fd);
+		}
 
 		/* grab a lock, again to call callbacks and update status. */
 		rte_spinlock_lock(&intr_lock);
diff --git a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h
index 45071b7..b095b86 100644
--- a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h
+++ b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h
@@ -48,6 +48,7 @@ enum rte_intr_handle_type {
 	RTE_INTR_HANDLE_VFIO_MSI,     /**< vfio device handle (MSI) */
 	RTE_INTR_HANDLE_VFIO_MSIX,    /**< vfio device handle (MSIX) */
 	RTE_INTR_HANDLE_ALARM,    /**< alarm handle */
+	RTE_INTR_HANDLE_EXT, /**< external handler */
 	RTE_INTR_HANDLE_MAX
 };
 
-- 
2.1.0

[dpdk-dev] [PATCH 2/3] mlx4: handle interrupts

[Adrien Mazarguil]

From: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

Add interrupts handler for port status notification.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx4/mlx4.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx4/mlx4.h |   3 +
 2 files changed, 169 insertions(+)

diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index af31573..cb08ee8 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -57,6 +57,7 @@
 #include <linux/if.h>
 #include <linux/ethtool.h>
 #include <linux/sockios.h>
+#include <fcntl.h>
 
 /* Verbs header. */
 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
@@ -85,6 +86,7 @@
 #include <rte_atomic.h>
 #include <rte_version.h>
 #include <rte_log.h>
+#include <rte_alarm.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-pedantic"
 #endif
@@ -282,6 +284,7 @@ struct priv {
 	unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */
 	unsigned int rss:1; /* RSS is enabled. */
 	unsigned int vf:1; /* This is a VF device. */
+	unsigned int pending_alarm:1; /* An alarm is pending. */
 #ifdef INLINE_RECV
 	unsigned int inl_recv_size; /* Inline recv size */
 #endif
@@ -292,6 +295,7 @@ struct priv {
 	unsigned int txqs_n; /* TX queues array size. */
 	struct rxq *(*rxqs)[]; /* RX queues. */
 	struct txq *(*txqs)[]; /* TX queues. */
+	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	rte_spinlock_t lock; /* Lock for control functions. */
 };
 
@@ -3585,6 +3589,9 @@ mlx4_rx_queue_release(void *dpdk_rxq)
 	priv_unlock(priv);
 }
 
+static void
+priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *);
+
 /**
  * DPDK callback to start the device.
  *
@@ -3646,6 +3653,7 @@ mlx4_dev_start(struct rte_eth_dev *dev)
 		priv->started = 0;
 		return -ret;
 	} while ((--r) && ((rxq = (*priv->rxqs)[++i]), i));
+	priv_dev_interrupt_handler_install(priv, dev);
 	priv_unlock(priv);
 	return 0;
 }
@@ -3742,6 +3750,9 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	return 0;
 }
 
+static void
+priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
+
 /**
  * DPDK callback to close the device.
  *
@@ -3802,6 +3813,7 @@ mlx4_dev_close(struct rte_eth_dev *dev)
 		claim_zero(ibv_close_device(priv->ctx));
 	} else
 		assert(priv->ctx == NULL);
+	priv_dev_interrupt_handler_uninstall(priv, dev);
 	priv_unlock(priv);
 	memset(priv, 0, sizeof(*priv));
 }
@@ -4693,6 +4705,158 @@ mlx4_getenv_int(const char *name)
 	return atoi(val);
 }
 
+static void
+mlx4_dev_link_status_handler(void *);
+static void
+mlx4_dev_interrupt_handler(struct rte_intr_handle *, void *);
+
+/**
+ * Link status handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ *
+ * @return
+ *   Nonzero if the callback process can be called immediately.
+ */
+static int
+priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev)
+{
+	struct ibv_async_event event;
+	int port_change = 0;
+	int ret = 0;
+
+	/* Read all message and acknowledge them. */
+	for (;;) {
+		if (ibv_get_async_event(priv->ctx, &event))
+			break;
+
+		if (event.event_type == IBV_EVENT_PORT_ACTIVE ||
+		    event.event_type == IBV_EVENT_PORT_ERR)
+			port_change = 1;
+		else
+			DEBUG("event type %d on port %d not handled",
+			      event.event_type, event.element.port_num);
+		ibv_ack_async_event(&event);
+	}
+
+	if (port_change ^ priv->pending_alarm) {
+		struct rte_eth_link *link = &dev->data->dev_link;
+
+		priv->pending_alarm = 0;
+		mlx4_link_update_unlocked(dev, 0);
+		if (((link->link_speed == 0) && link->link_status) ||
+		    ((link->link_speed != 0) && !link->link_status)) {
+			/* Inconsistent status, check again later. */
+			priv->pending_alarm = 1;
+			rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US,
+					  mlx4_dev_link_status_handler,
+					  dev);
+		} else
+			ret = 1;
+	}
+	return ret;
+}
+
+/**
+ * Handle delayed link status event.
+ *
+ * @param arg
+ *   Registered argument.
+ */
+static void
+mlx4_dev_link_status_handler(void *arg)
+{
+	struct rte_eth_dev *dev = arg;
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	priv_lock(priv);
+	assert(priv->pending_alarm == 1);
+	ret = priv_dev_link_status_handler(priv, dev);
+	priv_unlock(priv);
+	if (ret)
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Handle interrupts from the NIC.
+ *
+ * @param[in] intr_handle
+ *   Interrupt handler.
+ * @param cb_arg
+ *   Callback argument.
+ */
+static void
+mlx4_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg)
+{
+	struct rte_eth_dev *dev = cb_arg;
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	(void)intr_handle;
+	priv_lock(priv);
+	ret = priv_dev_link_status_handler(priv, dev);
+	priv_unlock(priv);
+	if (ret)
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Uninstall interrupt handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ */
+static void
+priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
+{
+	if (!dev->data->dev_conf.intr_conf.lsc)
+		return;
+	rte_intr_callback_unregister(&priv->intr_handle,
+				     mlx4_dev_interrupt_handler,
+				     dev);
+	if (priv->pending_alarm)
+		rte_eal_alarm_cancel(mlx4_dev_link_status_handler, dev);
+	priv->pending_alarm = 0;
+	priv->intr_handle.fd = 0;
+	priv->intr_handle.type = 0;
+}
+
+/**
+ * Install interrupt handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ */
+static void
+priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
+{
+	int rc, flags;
+
+	if (!dev->data->dev_conf.intr_conf.lsc)
+		return;
+	assert(priv->ctx->async_fd > 0);
+	flags = fcntl(priv->ctx->async_fd, F_GETFL);
+	rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+	if (rc < 0) {
+		INFO("failed to change file descriptor async event queue");
+		dev->data->dev_conf.intr_conf.lsc = 0;
+	} else {
+		priv->intr_handle.fd = priv->ctx->async_fd;
+		priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
+		rte_intr_callback_register(&priv->intr_handle,
+					   mlx4_dev_interrupt_handler,
+					   dev);
+	}
+}
+
 static struct eth_driver mlx4_driver;
 
 /**
@@ -4979,6 +5143,7 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		priv->dev = eth_dev;
 		eth_dev->dev_ops = &mlx4_dev_ops;
 		eth_dev->data->mac_addrs = priv->mac;
+		TAILQ_INIT(&eth_dev->link_intr_cbs);
 
 		/* Bring Ethernet device up. */
 		DEBUG("forcing Ethernet interface up");
@@ -5045,6 +5210,7 @@ static struct eth_driver mlx4_driver = {
 		.name = MLX4_DRIVER_NAME,
 		.id_table = mlx4_pci_id_map,
 		.devinit = mlx4_pci_devinit,
+		.drv_flags = RTE_PCI_DRV_INTR_LSC,
 	},
 	.dev_private_size = sizeof(struct priv)
 };
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 151c34b..d0c7bc2 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -83,6 +83,9 @@
 #define MLX4_PMD_SOFT_COUNTERS 1
 #endif
 
+/* Alarm timeout. */
+#define MLX4_ALARM_TIMEOUT_US 100000
+
 enum {
 	PCI_VENDOR_ID_MELLANOX = 0x15b3,
 };
-- 
2.1.0

[dpdk-dev] [PATCH 3/3] mlx5: handle interrupts

[Adrien Mazarguil]

From: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

Add interrupts handler for port status notification.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 drivers/net/mlx5/mlx5.c         |   3 +
 drivers/net/mlx5/mlx5.h         |   7 ++
 drivers/net/mlx5/mlx5_defs.h    |   3 +
 drivers/net/mlx5/mlx5_ethdev.c  | 150 ++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_trigger.c |   4 ++
 5 files changed, 167 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index fb9d594..7cf533f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -87,6 +87,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	      ((priv->ctx != NULL) ? priv->ctx->device->name : ""));
 	/* In case mlx5_dev_stop() has not been called. */
 	if (priv->started) {
+		priv_dev_interrupt_handler_uninstall(priv, dev);
 		priv_allmulticast_disable(priv);
 		priv_promiscuous_disable(priv);
 		priv_mac_addrs_disable(priv);
@@ -452,6 +453,7 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		priv->dev = eth_dev;
 		eth_dev->dev_ops = &mlx5_dev_ops;
 		eth_dev->data->mac_addrs = priv->mac;
+		TAILQ_INIT(&eth_dev->link_intr_cbs);
 
 		/* Bring Ethernet device up. */
 		DEBUG("forcing Ethernet interface up");
@@ -525,6 +527,7 @@ static struct eth_driver mlx5_driver = {
 		.name = MLX5_DRIVER_NAME,
 		.id_table = mlx5_pci_id_map,
 		.devinit = mlx5_pci_devinit,
+		.drv_flags = RTE_PCI_DRV_INTR_LSC,
 	},
 	.dev_private_size = sizeof(struct priv)
 };
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index de72f94..6e6bc3a 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -58,6 +58,7 @@
 #include <rte_ether.h>
 #include <rte_ethdev.h>
 #include <rte_spinlock.h>
+#include <rte_interrupts.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-pedantic"
 #endif
@@ -106,6 +107,7 @@ struct priv {
 	unsigned int hw_csum:1; /* Checksum offload is supported. */
 	unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */
 	unsigned int vf:1; /* This is a VF device. */
+	unsigned int pending_alarm:1; /* An alarm is pending. */
 	/* RX/TX queues. */
 	unsigned int rxqs_n; /* RX queues array size. */
 	unsigned int txqs_n; /* TX queues array size. */
@@ -120,6 +122,7 @@ struct priv {
 	unsigned int hash_rxqs_n; /* Hash RX QPs array size. */
 	/* RSS configuration array indexed by hash RX queue type. */
 	struct rte_eth_rss_conf *(*rss_conf)[];
+	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	rte_spinlock_t lock; /* Lock for control functions. */
 };
 
@@ -162,6 +165,10 @@ int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *);
 int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *);
 int mlx5_ibv_device_to_pci_addr(const struct ibv_device *,
 				struct rte_pci_addr *);
+void mlx5_dev_link_status_handler(void *);
+void mlx5_dev_interrupt_handler(struct rte_intr_handle *, void *);
+void priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
+void priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *);
 
 /* mlx5_mac.c */
 
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index e697764..3298e94 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -85,4 +85,7 @@
 #define MLX5_PMD_SOFT_COUNTERS 1
 #endif
 
+/* Alarm timeout. */
+#define MLX5_ALARM_TIMEOUT_US 100000
+
 #endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index fac685e..84e877c 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -47,6 +47,7 @@
 #include <linux/if.h>
 #include <linux/ethtool.h>
 #include <linux/sockios.h>
+#include <fcntl.h>
 
 /* DPDK headers don't like -pedantic. */
 #ifdef PEDANTIC
@@ -56,6 +57,8 @@
 #include <rte_ethdev.h>
 #include <rte_mbuf.h>
 #include <rte_common.h>
+#include <rte_interrupts.h>
+#include <rte_alarm.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-pedantic"
 #endif
@@ -790,3 +793,150 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
 	fclose(file);
 	return 0;
 }
+
+/**
+ * Link status handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ *
+ * @return
+ *   Nonzero if the callback process can be called immediately.
+ */
+static int
+priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev)
+{
+	struct ibv_async_event event;
+	int port_change = 0;
+	int ret = 0;
+
+	/* Read all message and acknowledge them. */
+	for (;;) {
+		if (ibv_get_async_event(priv->ctx, &event))
+			break;
+
+		if (event.event_type == IBV_EVENT_PORT_ACTIVE ||
+		    event.event_type == IBV_EVENT_PORT_ERR)
+			port_change = 1;
+		else
+			DEBUG("event type %d on port %d not handled",
+			      event.event_type, event.element.port_num);
+		ibv_ack_async_event(&event);
+	}
+
+	if (port_change ^ priv->pending_alarm) {
+		struct rte_eth_link *link = &dev->data->dev_link;
+
+		priv->pending_alarm = 0;
+		mlx5_link_update_unlocked(dev, 0);
+		if (((link->link_speed == 0) && link->link_status) ||
+		    ((link->link_speed != 0) && !link->link_status)) {
+			/* Inconsistent status, check again later. */
+			priv->pending_alarm = 1;
+			rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US,
+					  mlx5_dev_link_status_handler,
+					  dev);
+		} else
+			ret = 1;
+	}
+	return ret;
+}
+
+/**
+ * Handle delayed link status event.
+ *
+ * @param arg
+ *   Registered argument.
+ */
+void
+mlx5_dev_link_status_handler(void *arg)
+{
+	struct rte_eth_dev *dev = arg;
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	priv_lock(priv);
+	assert(priv->pending_alarm == 1);
+	ret = priv_dev_link_status_handler(priv, dev);
+	priv_unlock(priv);
+	if (ret)
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Handle interrupts from the NIC.
+ *
+ * @param[in] intr_handle
+ *   Interrupt handler.
+ * @param cb_arg
+ *   Callback argument.
+ */
+void
+mlx5_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg)
+{
+	struct rte_eth_dev *dev = cb_arg;
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	(void)intr_handle;
+	priv_lock(priv);
+	ret = priv_dev_link_status_handler(priv, dev);
+	priv_unlock(priv);
+	if (ret)
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Uninstall interrupt handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ */
+void
+priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
+{
+	if (!dev->data->dev_conf.intr_conf.lsc)
+		return;
+	rte_intr_callback_unregister(&priv->intr_handle,
+				     mlx5_dev_interrupt_handler,
+				     dev);
+	if (priv->pending_alarm)
+		rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev);
+	priv->pending_alarm = 0;
+	priv->intr_handle.fd = 0;
+	priv->intr_handle.type = 0;
+}
+
+/**
+ * Install interrupt handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ */
+void
+priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
+{
+	int rc, flags;
+
+	if (!dev->data->dev_conf.intr_conf.lsc)
+		return;
+	assert(priv->ctx->async_fd > 0);
+	flags = fcntl(priv->ctx->async_fd, F_GETFL);
+	rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+	if (rc < 0) {
+		INFO("failed to change file descriptor async event queue");
+		dev->data->dev_conf.intr_conf.lsc = 0;
+	} else {
+		priv->intr_handle.fd = priv->ctx->async_fd;
+		priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
+		rte_intr_callback_register(&priv->intr_handle,
+					   mlx5_dev_interrupt_handler,
+					   dev);
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 233c0d8..b0a6d24 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -37,6 +37,8 @@
 #endif
 #include <rte_ether.h>
 #include <rte_ethdev.h>
+#include <rte_interrupts.h>
+#include <rte_alarm.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-pedantic"
 #endif
@@ -87,6 +89,7 @@ mlx5_dev_start(struct rte_eth_dev *dev)
 		priv_mac_addrs_disable(priv);
 		priv_destroy_hash_rxqs(priv);
 	}
+	priv_dev_interrupt_handler_install(priv, dev);
 	priv_unlock(priv);
 	return -err;
 }
@@ -114,6 +117,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	priv_promiscuous_disable(priv);
 	priv_mac_addrs_disable(priv);
 	priv_destroy_hash_rxqs(priv);
+	priv_dev_interrupt_handler_uninstall(priv, dev);
 	priv->started = 0;
 	priv_unlock(priv);
 }
-- 
2.1.0

Re: [dpdk-dev] [PATCH 1/3] eal: new interrupt handler type

[David Marchand]

On Mon, Oct 5, 2015 at 7:56 PM, Adrien Mazarguil <adrien.mazarguil@6wind.com
> wrote:

> From: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
>
> Add RTE_INTR_HANDLE_EXT handler type for PMDs that do not support VFIO or
> UIO.  Those are expected to manage the file descriptor themselves.
>
> Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
>

Maybe a title like "eal: allow external interrupt handler" would be better.
Acked-by: David Marchand <david.marchand@6wind.com>

-- 
David Marchand

[dpdk-dev] [PATCH v2 0/3] Add link status notification support to Mellanox PMDs

[Adrien Mazarguil]

Mellanox PMDs (mlx4 and mlx5) support link status notification with
libibverbs through a file descriptor, in a manner reminiscent of UIO/VFIO.

It still requires a new "external" interrupt handler type since contrary
to other types, this file descriptor must not be processed by EAL.

Changes in v2:
- Only changes caused by rebase on v2 of previous patchsets
  ("Mellanox ConnectX-4 PMD (mlx5)" and
  "Enhance mlx5 with Mellanox OFED 3.1")

Nelio Laranjeiro (3):
  eal: new interrupt handler type
  mlx4: handle interrupts
  mlx5: handle interrupts

 drivers/net/mlx4/mlx4.c                            | 166 +++++++++++++++++++++
 drivers/net/mlx4/mlx4.h                            |   3 +
 drivers/net/mlx5/mlx5.c                            |   3 +
 drivers/net/mlx5/mlx5.h                            |   7 +
 drivers/net/mlx5/mlx5_defs.h                       |   3 +
 drivers/net/mlx5/mlx5_ethdev.c                     | 150 +++++++++++++++++++
 drivers/net/mlx5/mlx5_trigger.c                    |   4 +
 lib/librte_eal/linuxapp/eal/eal_interrupts.c       |  34 +++--
 .../linuxapp/eal/include/exec-env/rte_interrupts.h |   1 +
 9 files changed, 356 insertions(+), 15 deletions(-)

-- 
2.1.0

[dpdk-dev] [PATCH v2 1/3] eal: new interrupt handler type

[Adrien Mazarguil]

From: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

Add RTE_INTR_HANDLE_EXT handler type for PMDs that do not support VFIO or
UIO.  Those are expected to manage the file descriptor themselves.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 lib/librte_eal/linuxapp/eal/eal_interrupts.c       | 34 ++++++++++++----------
 .../linuxapp/eal/include/exec-env/rte_interrupts.h |  1 +
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
index 078318c..79eedaf 100644
--- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
@@ -691,26 +691,30 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 			bytes_read = sizeof(buf.vfio_intr_count);
 			break;
 #endif
+		case RTE_INTR_HANDLE_EXT:
 		default:
 			bytes_read = 1;
 			break;
 		}
 
-		/**
-		 * read out to clear the ready-to-be-read flag
-		 * for epoll_wait.
-		 */
-		bytes_read = read(events[n].data.fd, &buf, bytes_read);
-		if (bytes_read < 0) {
-			if (errno == EINTR || errno == EWOULDBLOCK)
-				continue;
-
-			RTE_LOG(ERR, EAL, "Error reading from file "
-				"descriptor %d: %s\n", events[n].data.fd,
-							strerror(errno));
-		} else if (bytes_read == 0)
-			RTE_LOG(ERR, EAL, "Read nothing from file "
-				"descriptor %d\n", events[n].data.fd);
+		if (src->intr_handle.type != RTE_INTR_HANDLE_EXT) {
+			/**
+			 * read out to clear the ready-to-be-read flag
+			 * for epoll_wait.
+			 */
+			bytes_read = read(events[n].data.fd, &buf, bytes_read);
+			if (bytes_read < 0) {
+				if (errno == EINTR || errno == EWOULDBLOCK)
+					continue;
+
+				RTE_LOG(ERR, EAL, "Error reading from file "
+					"descriptor %d: %s\n",
+					events[n].data.fd,
+					strerror(errno));
+			} else if (bytes_read == 0)
+				RTE_LOG(ERR, EAL, "Read nothing from file "
+					"descriptor %d\n", events[n].data.fd);
+		}
 
 		/* grab a lock, again to call callbacks and update status. */
 		rte_spinlock_lock(&intr_lock);
diff --git a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h
index 45071b7..b095b86 100644
--- a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h
+++ b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_interrupts.h
@@ -48,6 +48,7 @@ enum rte_intr_handle_type {
 	RTE_INTR_HANDLE_VFIO_MSI,     /**< vfio device handle (MSI) */
 	RTE_INTR_HANDLE_VFIO_MSIX,    /**< vfio device handle (MSIX) */
 	RTE_INTR_HANDLE_ALARM,    /**< alarm handle */
+	RTE_INTR_HANDLE_EXT, /**< external handler */
 	RTE_INTR_HANDLE_MAX
 };
 
-- 
2.1.0

[dpdk-dev] [PATCH v2 2/3] mlx4: handle interrupts

[Adrien Mazarguil]

From: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

Add interrupts handler for port status notification.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx4/mlx4.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx4/mlx4.h |   3 +
 2 files changed, 169 insertions(+)

diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index af31573..cb08ee8 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -57,6 +57,7 @@
 #include <linux/if.h>
 #include <linux/ethtool.h>
 #include <linux/sockios.h>
+#include <fcntl.h>
 
 /* Verbs header. */
 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
@@ -85,6 +86,7 @@
 #include <rte_atomic.h>
 #include <rte_version.h>
 #include <rte_log.h>
+#include <rte_alarm.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-pedantic"
 #endif
@@ -282,6 +284,7 @@ struct priv {
 	unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */
 	unsigned int rss:1; /* RSS is enabled. */
 	unsigned int vf:1; /* This is a VF device. */
+	unsigned int pending_alarm:1; /* An alarm is pending. */
 #ifdef INLINE_RECV
 	unsigned int inl_recv_size; /* Inline recv size */
 #endif
@@ -292,6 +295,7 @@ struct priv {
 	unsigned int txqs_n; /* TX queues array size. */
 	struct rxq *(*rxqs)[]; /* RX queues. */
 	struct txq *(*txqs)[]; /* TX queues. */
+	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	rte_spinlock_t lock; /* Lock for control functions. */
 };
 
@@ -3585,6 +3589,9 @@ mlx4_rx_queue_release(void *dpdk_rxq)
 	priv_unlock(priv);
 }
 
+static void
+priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *);
+
 /**
  * DPDK callback to start the device.
  *
@@ -3646,6 +3653,7 @@ mlx4_dev_start(struct rte_eth_dev *dev)
 		priv->started = 0;
 		return -ret;
 	} while ((--r) && ((rxq = (*priv->rxqs)[++i]), i));
+	priv_dev_interrupt_handler_install(priv, dev);
 	priv_unlock(priv);
 	return 0;
 }
@@ -3742,6 +3750,9 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	return 0;
 }
 
+static void
+priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
+
 /**
  * DPDK callback to close the device.
  *
@@ -3802,6 +3813,7 @@ mlx4_dev_close(struct rte_eth_dev *dev)
 		claim_zero(ibv_close_device(priv->ctx));
 	} else
 		assert(priv->ctx == NULL);
+	priv_dev_interrupt_handler_uninstall(priv, dev);
 	priv_unlock(priv);
 	memset(priv, 0, sizeof(*priv));
 }
@@ -4693,6 +4705,158 @@ mlx4_getenv_int(const char *name)
 	return atoi(val);
 }
 
+static void
+mlx4_dev_link_status_handler(void *);
+static void
+mlx4_dev_interrupt_handler(struct rte_intr_handle *, void *);
+
+/**
+ * Link status handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ *
+ * @return
+ *   Nonzero if the callback process can be called immediately.
+ */
+static int
+priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev)
+{
+	struct ibv_async_event event;
+	int port_change = 0;
+	int ret = 0;
+
+	/* Read all message and acknowledge them. */
+	for (;;) {
+		if (ibv_get_async_event(priv->ctx, &event))
+			break;
+
+		if (event.event_type == IBV_EVENT_PORT_ACTIVE ||
+		    event.event_type == IBV_EVENT_PORT_ERR)
+			port_change = 1;
+		else
+			DEBUG("event type %d on port %d not handled",
+			      event.event_type, event.element.port_num);
+		ibv_ack_async_event(&event);
+	}
+
+	if (port_change ^ priv->pending_alarm) {
+		struct rte_eth_link *link = &dev->data->dev_link;
+
+		priv->pending_alarm = 0;
+		mlx4_link_update_unlocked(dev, 0);
+		if (((link->link_speed == 0) && link->link_status) ||
+		    ((link->link_speed != 0) && !link->link_status)) {
+			/* Inconsistent status, check again later. */
+			priv->pending_alarm = 1;
+			rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US,
+					  mlx4_dev_link_status_handler,
+					  dev);
+		} else
+			ret = 1;
+	}
+	return ret;
+}
+
+/**
+ * Handle delayed link status event.
+ *
+ * @param arg
+ *   Registered argument.
+ */
+static void
+mlx4_dev_link_status_handler(void *arg)
+{
+	struct rte_eth_dev *dev = arg;
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	priv_lock(priv);
+	assert(priv->pending_alarm == 1);
+	ret = priv_dev_link_status_handler(priv, dev);
+	priv_unlock(priv);
+	if (ret)
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Handle interrupts from the NIC.
+ *
+ * @param[in] intr_handle
+ *   Interrupt handler.
+ * @param cb_arg
+ *   Callback argument.
+ */
+static void
+mlx4_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg)
+{
+	struct rte_eth_dev *dev = cb_arg;
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	(void)intr_handle;
+	priv_lock(priv);
+	ret = priv_dev_link_status_handler(priv, dev);
+	priv_unlock(priv);
+	if (ret)
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Uninstall interrupt handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ */
+static void
+priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
+{
+	if (!dev->data->dev_conf.intr_conf.lsc)
+		return;
+	rte_intr_callback_unregister(&priv->intr_handle,
+				     mlx4_dev_interrupt_handler,
+				     dev);
+	if (priv->pending_alarm)
+		rte_eal_alarm_cancel(mlx4_dev_link_status_handler, dev);
+	priv->pending_alarm = 0;
+	priv->intr_handle.fd = 0;
+	priv->intr_handle.type = 0;
+}
+
+/**
+ * Install interrupt handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ */
+static void
+priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
+{
+	int rc, flags;
+
+	if (!dev->data->dev_conf.intr_conf.lsc)
+		return;
+	assert(priv->ctx->async_fd > 0);
+	flags = fcntl(priv->ctx->async_fd, F_GETFL);
+	rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+	if (rc < 0) {
+		INFO("failed to change file descriptor async event queue");
+		dev->data->dev_conf.intr_conf.lsc = 0;
+	} else {
+		priv->intr_handle.fd = priv->ctx->async_fd;
+		priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
+		rte_intr_callback_register(&priv->intr_handle,
+					   mlx4_dev_interrupt_handler,
+					   dev);
+	}
+}
+
 static struct eth_driver mlx4_driver;
 
 /**
@@ -4979,6 +5143,7 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		priv->dev = eth_dev;
 		eth_dev->dev_ops = &mlx4_dev_ops;
 		eth_dev->data->mac_addrs = priv->mac;
+		TAILQ_INIT(&eth_dev->link_intr_cbs);
 
 		/* Bring Ethernet device up. */
 		DEBUG("forcing Ethernet interface up");
@@ -5045,6 +5210,7 @@ static struct eth_driver mlx4_driver = {
 		.name = MLX4_DRIVER_NAME,
 		.id_table = mlx4_pci_id_map,
 		.devinit = mlx4_pci_devinit,
+		.drv_flags = RTE_PCI_DRV_INTR_LSC,
 	},
 	.dev_private_size = sizeof(struct priv)
 };
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 151c34b..d0c7bc2 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -83,6 +83,9 @@
 #define MLX4_PMD_SOFT_COUNTERS 1
 #endif
 
+/* Alarm timeout. */
+#define MLX4_ALARM_TIMEOUT_US 100000
+
 enum {
 	PCI_VENDOR_ID_MELLANOX = 0x15b3,
 };
-- 
2.1.0

[dpdk-dev] [PATCH v2 3/3] mlx5: handle interrupts

[Adrien Mazarguil]

From: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

Add interrupts handler for port status notification.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Signed-off-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
---
 drivers/net/mlx5/mlx5.c         |   3 +
 drivers/net/mlx5/mlx5.h         |   7 ++
 drivers/net/mlx5/mlx5_defs.h    |   3 +
 drivers/net/mlx5/mlx5_ethdev.c  | 150 ++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_trigger.c |   4 ++
 5 files changed, 167 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 97ce902..9636588 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -86,6 +86,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	      (void *)dev,
 	      ((priv->ctx != NULL) ? priv->ctx->device->name : ""));
 	/* In case mlx5_dev_stop() has not been called. */
+	priv_dev_interrupt_handler_uninstall(priv, dev);
 	priv_allmulticast_disable(priv);
 	priv_promiscuous_disable(priv);
 	priv_mac_addrs_disable(priv);
@@ -450,6 +451,7 @@ mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		priv->dev = eth_dev;
 		eth_dev->dev_ops = &mlx5_dev_ops;
 		eth_dev->data->mac_addrs = priv->mac;
+		TAILQ_INIT(&eth_dev->link_intr_cbs);
 
 		/* Bring Ethernet device up. */
 		DEBUG("forcing Ethernet interface up");
@@ -523,6 +525,7 @@ static struct eth_driver mlx5_driver = {
 		.name = MLX5_DRIVER_NAME,
 		.id_table = mlx5_pci_id_map,
 		.devinit = mlx5_pci_devinit,
+		.drv_flags = RTE_PCI_DRV_INTR_LSC,
 	},
 	.dev_private_size = sizeof(struct priv)
 };
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 03e33d6..0daacc8 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -58,6 +58,7 @@
 #include <rte_ether.h>
 #include <rte_ethdev.h>
 #include <rte_spinlock.h>
+#include <rte_interrupts.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-pedantic"
 #endif
@@ -101,6 +102,7 @@ struct priv {
 	unsigned int hw_csum:1; /* Checksum offload is supported. */
 	unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */
 	unsigned int vf:1; /* This is a VF device. */
+	unsigned int pending_alarm:1; /* An alarm is pending. */
 	/* RX/TX queues. */
 	unsigned int rxqs_n; /* RX queues array size. */
 	unsigned int txqs_n; /* TX queues array size. */
@@ -115,6 +117,7 @@ struct priv {
 	unsigned int hash_rxqs_n; /* Hash RX QPs array size. */
 	/* RSS configuration array indexed by hash RX queue type. */
 	struct rte_eth_rss_conf *(*rss_conf)[];
+	struct rte_intr_handle intr_handle; /* Interrupt handler. */
 	rte_spinlock_t lock; /* Lock for control functions. */
 };
 
@@ -157,6 +160,10 @@ int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *);
 int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *);
 int mlx5_ibv_device_to_pci_addr(const struct ibv_device *,
 				struct rte_pci_addr *);
+void mlx5_dev_link_status_handler(void *);
+void mlx5_dev_interrupt_handler(struct rte_intr_handle *, void *);
+void priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
+void priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *);
 
 /* mlx5_mac.c */
 
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 3952c71..bb82c9a 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -78,4 +78,7 @@
 #define MLX5_PMD_SOFT_COUNTERS 1
 #endif
 
+/* Alarm timeout. */
+#define MLX5_ALARM_TIMEOUT_US 100000
+
 #endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index fac685e..84e877c 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -47,6 +47,7 @@
 #include <linux/if.h>
 #include <linux/ethtool.h>
 #include <linux/sockios.h>
+#include <fcntl.h>
 
 /* DPDK headers don't like -pedantic. */
 #ifdef PEDANTIC
@@ -56,6 +57,8 @@
 #include <rte_ethdev.h>
 #include <rte_mbuf.h>
 #include <rte_common.h>
+#include <rte_interrupts.h>
+#include <rte_alarm.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-pedantic"
 #endif
@@ -790,3 +793,150 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
 	fclose(file);
 	return 0;
 }
+
+/**
+ * Link status handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ *
+ * @return
+ *   Nonzero if the callback process can be called immediately.
+ */
+static int
+priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev)
+{
+	struct ibv_async_event event;
+	int port_change = 0;
+	int ret = 0;
+
+	/* Read all message and acknowledge them. */
+	for (;;) {
+		if (ibv_get_async_event(priv->ctx, &event))
+			break;
+
+		if (event.event_type == IBV_EVENT_PORT_ACTIVE ||
+		    event.event_type == IBV_EVENT_PORT_ERR)
+			port_change = 1;
+		else
+			DEBUG("event type %d on port %d not handled",
+			      event.event_type, event.element.port_num);
+		ibv_ack_async_event(&event);
+	}
+
+	if (port_change ^ priv->pending_alarm) {
+		struct rte_eth_link *link = &dev->data->dev_link;
+
+		priv->pending_alarm = 0;
+		mlx5_link_update_unlocked(dev, 0);
+		if (((link->link_speed == 0) && link->link_status) ||
+		    ((link->link_speed != 0) && !link->link_status)) {
+			/* Inconsistent status, check again later. */
+			priv->pending_alarm = 1;
+			rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US,
+					  mlx5_dev_link_status_handler,
+					  dev);
+		} else
+			ret = 1;
+	}
+	return ret;
+}
+
+/**
+ * Handle delayed link status event.
+ *
+ * @param arg
+ *   Registered argument.
+ */
+void
+mlx5_dev_link_status_handler(void *arg)
+{
+	struct rte_eth_dev *dev = arg;
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	priv_lock(priv);
+	assert(priv->pending_alarm == 1);
+	ret = priv_dev_link_status_handler(priv, dev);
+	priv_unlock(priv);
+	if (ret)
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Handle interrupts from the NIC.
+ *
+ * @param[in] intr_handle
+ *   Interrupt handler.
+ * @param cb_arg
+ *   Callback argument.
+ */
+void
+mlx5_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg)
+{
+	struct rte_eth_dev *dev = cb_arg;
+	struct priv *priv = dev->data->dev_private;
+	int ret;
+
+	(void)intr_handle;
+	priv_lock(priv);
+	ret = priv_dev_link_status_handler(priv, dev);
+	priv_unlock(priv);
+	if (ret)
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+}
+
+/**
+ * Uninstall interrupt handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ */
+void
+priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
+{
+	if (!dev->data->dev_conf.intr_conf.lsc)
+		return;
+	rte_intr_callback_unregister(&priv->intr_handle,
+				     mlx5_dev_interrupt_handler,
+				     dev);
+	if (priv->pending_alarm)
+		rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev);
+	priv->pending_alarm = 0;
+	priv->intr_handle.fd = 0;
+	priv->intr_handle.type = 0;
+}
+
+/**
+ * Install interrupt handler.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ * @param dev
+ *   Pointer to the rte_eth_dev structure.
+ */
+void
+priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
+{
+	int rc, flags;
+
+	if (!dev->data->dev_conf.intr_conf.lsc)
+		return;
+	assert(priv->ctx->async_fd > 0);
+	flags = fcntl(priv->ctx->async_fd, F_GETFL);
+	rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+	if (rc < 0) {
+		INFO("failed to change file descriptor async event queue");
+		dev->data->dev_conf.intr_conf.lsc = 0;
+	} else {
+		priv->intr_handle.fd = priv->ctx->async_fd;
+		priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
+		rte_intr_callback_register(&priv->intr_handle,
+					   mlx5_dev_interrupt_handler,
+					   dev);
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 68e00a0..ff1203d 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -37,6 +37,8 @@
 #endif
 #include <rte_ether.h>
 #include <rte_ethdev.h>
+#include <rte_interrupts.h>
+#include <rte_alarm.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-pedantic"
 #endif
@@ -87,6 +89,7 @@ mlx5_dev_start(struct rte_eth_dev *dev)
 		priv_mac_addrs_disable(priv);
 		priv_destroy_hash_rxqs(priv);
 	}
+	priv_dev_interrupt_handler_install(priv, dev);
 	priv_unlock(priv);
 	return -err;
 }
@@ -114,6 +117,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	priv_promiscuous_disable(priv);
 	priv_mac_addrs_disable(priv);
 	priv_destroy_hash_rxqs(priv);
+	priv_dev_interrupt_handler_uninstall(priv, dev);
 	priv->started = 0;
 	priv_unlock(priv);
 }
-- 
2.1.0

Re: [dpdk-dev] [PATCH v2 0/3] Add link status notification support to Mellanox PMDs

[Thomas Monjalon]

2015-10-30 19:57, Adrien Mazarguil:
> Mellanox PMDs (mlx4 and mlx5) support link status notification with
> libibverbs through a file descriptor, in a manner reminiscent of UIO/VFIO.
> 
> It still requires a new "external" interrupt handler type since contrary
> to other types, this file descriptor must not be processed by EAL.
> 
> Changes in v2:
> - Only changes caused by rebase on v2 of previous patchsets
>   ("Mellanox ConnectX-4 PMD (mlx5)" and
>   "Enhance mlx5 with Mellanox OFED 3.1")
> 
> Nelio Laranjeiro (3):
>   eal: new interrupt handler type
>   mlx4: handle interrupts
>   mlx5: handle interrupts

Applied with previous ack from David on v1 and suggested title change.
Please take care of addressing comments yourself.
Thanks

Re: [dpdk-dev] [PATCH v2 0/3] Add link status notification support to Mellanox PMDs

[Adrien Mazarguil]

On Sun, Nov 01, 2015 at 11:32:06AM +0100, Thomas Monjalon wrote:
> 2015-10-30 19:57, Adrien Mazarguil:
> > Mellanox PMDs (mlx4 and mlx5) support link status notification with
> > libibverbs through a file descriptor, in a manner reminiscent of UIO/VFIO.
> > 
> > It still requires a new "external" interrupt handler type since contrary
> > to other types, this file descriptor must not be processed by EAL.
> > 
> > Changes in v2:
> > - Only changes caused by rebase on v2 of previous patchsets
> >   ("Mellanox ConnectX-4 PMD (mlx5)" and
> >   "Enhance mlx5 with Mellanox OFED 3.1")
> > 
> > Nelio Laranjeiro (3):
> >   eal: new interrupt handler type
> >   mlx4: handle interrupts
> >   mlx5: handle interrupts
> 
> Applied with previous ack from David on v1 and suggested title change.
> Please take care of addressing comments yourself.
> Thanks

Thanks, sorry for the omission, I was a bit in a hurry. I fully agree with
the title change.

-- 
Adrien Mazarguil
6WIND