[dpdk-dev] [PATCH v3 13/27] bus/fslmc: cleanup the dpaa2 interrupt support

[Hemant Agrawal <hemant.agrawal@nxp.com>]

Signed-off-by: Hemant Agrawal <hemant.agrawal@nxp.com>
---
 drivers/bus/fslmc/Makefile                  |   1 +
 drivers/bus/fslmc/fslmc_vfio.c              | 108 +++++++++++++++++++++++-----
 drivers/bus/fslmc/fslmc_vfio.h              |   8 ++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c    |  18 +++--
 drivers/bus/fslmc/rte_bus_fslmc_version.map |   2 +
 5 files changed, 110 insertions(+), 27 deletions(-)

diff --git a/drivers/bus/fslmc/Makefile b/drivers/bus/fslmc/Makefile
index d1b790b..37da1b0 100644
--- a/drivers/bus/fslmc/Makefile
+++ b/drivers/bus/fslmc/Makefile
@@ -51,6 +51,7 @@ CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc
 CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/mc
 CFLAGS += -I$(RTE_SDK)/drivers/bus/fslmc/qbman/include
 CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common
 
 # versioning export map
 EXPORT_MAP := rte_bus_fslmc_version.map
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 3de5238..b2413c2 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -46,6 +46,7 @@
 #include <dirent.h>
 #include <sys/eventfd.h>
 
+#include <eal_filesystem.h>
 #include <rte_mbuf.h>
 #include <rte_ethdev.h>
 #include <rte_malloc.h>
@@ -335,36 +336,107 @@ static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj)
 
 #define IRQ_SET_BUF_LEN  (sizeof(struct vfio_irq_set) + sizeof(int))
 
-int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle,
-			  uint32_t index)
+int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, int index)
 {
-	struct vfio_irq_set *irq_set;
+	int len, ret;
 	char irq_set_buf[IRQ_SET_BUF_LEN];
-	int *fd_ptr, fd, ret;
+	struct vfio_irq_set *irq_set;
+	int *fd_ptr;
+
+	len = sizeof(irq_set_buf);
 
-	/* Prepare vfio_irq_set structure and SET the IRQ in VFIO */
-	/* Give the eventfd to VFIO */
-	fd = eventfd(0, 0);
 	irq_set = (struct vfio_irq_set *)irq_set_buf;
-	irq_set->argsz = sizeof(irq_set_buf);
+	irq_set->argsz = len;
 	irq_set->count = 1;
-	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
-			 VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->flags =
+		VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
 	irq_set->index = index;
 	irq_set->start = 0;
 	fd_ptr = (int *)&irq_set->data;
-	*fd_ptr = fd;
+	*fd_ptr = intr_handle->fd;
 
 	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
-	if (ret < 0) {
-		FSLMC_VFIO_LOG(ERR, "Unable to set IRQ in VFIO, ret: %d\n",
-			       ret);
-		return -1;
+	if (ret) {
+		RTE_LOG(ERR, EAL, "Error:dpaa2 SET IRQs fd=%d, err = %d(%s)\n",
+			intr_handle->fd, errno, strerror(errno));
+		return ret;
 	}
 
-	/* Set the FD and update the flags */
-	intr_handle->fd = fd;
-	return 0;
+	return ret;
+}
+
+int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index)
+{
+	struct vfio_irq_set *irq_set;
+	char irq_set_buf[IRQ_SET_BUF_LEN];
+	int len, ret;
+
+	len = sizeof(struct vfio_irq_set);
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = len;
+	irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = index;
+	irq_set->start = 0;
+	irq_set->count = 0;
+
+	ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (ret)
+		RTE_LOG(ERR, EAL,
+			"Error disabling dpaa2 interrupts for fd %d\n",
+			intr_handle->fd);
+
+	return ret;
+}
+
+/* set up interrupt support (but not enable interrupts) */
+int
+rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
+			  int vfio_dev_fd,
+			  int num_irqs)
+{
+	int i, ret;
+
+	/* start from MSI-X interrupt type */
+	for (i = 0; i < num_irqs; i++) {
+		struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
+		int fd = -1;
+
+		irq_info.index = i;
+
+		ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
+		if (ret < 0) {
+			FSLMC_VFIO_LOG(ERR,
+				       "cannot get IRQ(%d) info, error %i (%s)",
+				       i, errno, strerror(errno));
+			return -1;
+		}
+
+		/* if this vector cannot be used with eventfd,
+		 * fail if we explicitly
+		 * specified interrupt type, otherwise continue
+		 */
+		if ((irq_info.flags & VFIO_IRQ_INFO_EVENTFD) == 0)
+			continue;
+
+		/* set up an eventfd for interrupts */
+		fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+		if (fd < 0) {
+			FSLMC_VFIO_LOG(ERR,
+				       "cannot set up eventfd, error %i (%s)\n",
+				       errno, strerror(errno));
+			return -1;
+		}
+
+		intr_handle->fd = fd;
+		intr_handle->type = RTE_INTR_HANDLE_VFIO_MSI;
+		intr_handle->vfio_dev_fd = vfio_dev_fd;
+
+		return 0;
+	}
+
+	/* if we're here, we haven't found a suitable interrupt vector */
+	return -1;
 }
 
 /*
diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index edb86d0..5470a41 100644
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -64,8 +64,12 @@ typedef struct fslmc_vfio_container {
 	struct fslmc_vfio_group *group;
 } fslmc_vfio_container;
 
-int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle,
-			  uint32_t index);
+int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, int index);
+int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index);
+
+int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
+			      int vfio_dev_fd,
+			      int num_irqs);
 
 int fslmc_vfio_setup_group(void);
 int fslmc_vfio_process_group(void);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index 8db1f6c..ff41ce4 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -440,7 +440,6 @@ dpaa2_create_dpio_device(int vdev_fd,
 {
 	struct dpaa2_dpio_dev *dpio_dev;
 	struct vfio_region_info reg_info = { .argsz = sizeof(reg_info)};
-	int vfio_dev_fd;
 
 	if (obj_info->num_regions < NUM_DPIO_REGIONS) {
 		PMD_INIT_LOG(ERR, "ERROR, Not sufficient number "
@@ -457,14 +456,12 @@ dpaa2_create_dpio_device(int vdev_fd,
 
 	dpio_dev->dpio = NULL;
 	dpio_dev->hw_id = object_id;
-	dpio_dev->intr_handle.vfio_dev_fd = vdev_fd;
 	rte_atomic16_init(&dpio_dev->ref_count);
 	/* Using single portal  for all devices */
 	dpio_dev->mc_portal = rte_mcp_ptr_list[MC_PORTAL_INDEX];
 
 	reg_info.index = 0;
-	vfio_dev_fd = dpio_dev->intr_handle.vfio_dev_fd;
-	if (ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
+	if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
 		PMD_INIT_LOG(ERR, "vfio: error getting region info\n");
 		rte_free(dpio_dev);
 		return -1;
@@ -473,10 +470,10 @@ dpaa2_create_dpio_device(int vdev_fd,
 	dpio_dev->ce_size = reg_info.size;
 	dpio_dev->qbman_portal_ce_paddr = (uint64_t)mmap(NULL, reg_info.size,
 				PROT_WRITE | PROT_READ, MAP_SHARED,
-				vfio_dev_fd, reg_info.offset);
+				vdev_fd, reg_info.offset);
 
 	reg_info.index = 1;
-	if (ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
+	if (ioctl(vdev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
 		PMD_INIT_LOG(ERR, "vfio: error getting region info\n");
 		rte_free(dpio_dev);
 		return -1;
@@ -485,7 +482,7 @@ dpaa2_create_dpio_device(int vdev_fd,
 	dpio_dev->ci_size = reg_info.size;
 	dpio_dev->qbman_portal_ci_paddr = (uint64_t)mmap(NULL, reg_info.size,
 				PROT_WRITE | PROT_READ, MAP_SHARED,
-				vfio_dev_fd, reg_info.offset);
+				vdev_fd, reg_info.offset);
 
 	if (configure_dpio_qbman_swp(dpio_dev)) {
 		PMD_INIT_LOG(ERR,
@@ -497,6 +494,13 @@ dpaa2_create_dpio_device(int vdev_fd,
 
 	io_space_count++;
 	dpio_dev->index = io_space_count;
+
+	if (rte_dpaa2_vfio_setup_intr(&dpio_dev->intr_handle, vdev_fd, 1)) {
+		PMD_INIT_LOG(ERR, "Fail to setup interrupt for %d\n",
+			     dpio_dev->hw_id);
+		rte_free(dpio_dev);
+	}
+
 	TAILQ_INSERT_TAIL(&dpio_dev_list, dpio_dev, next);
 	PMD_INIT_LOG(DEBUG, "DPAA2: Added [dpio.%d]", object_id);
 
diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
index 7b25248..51a2ac6 100644
--- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
+++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
@@ -85,5 +85,7 @@ DPDK_17.11 {
 
 	dpaa2_dpbp_supported;
 	rte_dpaa2_dev_type;
+	rte_dpaa2_intr_disable;
+	rte_dpaa2_intr_enable;
 
 } DPDK_17.08;
-- 
2.7.4