[PATCH 20.11] raw/ifpga: fix interrupt handle allocation

[PATCH 20.11] raw/ifpga: fix interrupt handle allocation

[Wei Huang]

[ upstream commit 20659eb38099033b9aa4f7476a7ba7b5d3258569 ]

Allocate FPGA interrupt handle instance for each card.

Fixes: e0a1aaf ("raw/ifpga: introduce IRQ functions")
Cc: stable@dpdk.org

Signed-off-by: Wei Huang <wei.huang@intel.com>
Acked-by: Tianfei Zhang <tianfei.zhang@intel.com>
---
 drivers/raw/ifpga/ifpga_rawdev.c | 90 +++++++++++++++++++++++++++-------------
 drivers/raw/ifpga/ifpga_rawdev.h |  7 +++-
 2 files changed, 67 insertions(+), 30 deletions(-)

diff --git a/drivers/raw/ifpga/ifpga_rawdev.c b/drivers/raw/ifpga/ifpga_rawdev.c
index fb37313..e04b791 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.c
+++ b/drivers/raw/ifpga/ifpga_rawdev.c
@@ -71,10 +71,6 @@
 static int ifpga_monitor_refcnt;
 static pthread_t ifpga_monitor_start_thread;
 
-#define IFPGA_MAX_IRQ 12
-/* 0 for FME interrupt, others are reserved for AFU irq */
-static struct rte_intr_handle ifpga_irq_handle[IFPGA_MAX_IRQ];
-
 static struct ifpga_rawdev *
 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
 static int set_surprise_link_check_aer(
@@ -118,6 +114,7 @@ struct ifpga_rawdev *
 {
 	struct ifpga_rawdev *dev;
 	uint16_t dev_id;
+	int i = 0;
 
 	dev = ifpga_rawdev_get(rawdev);
 	if (dev != NULL) {
@@ -134,6 +131,8 @@ struct ifpga_rawdev *
 	dev = &ifpga_rawdevices[dev_id];
 	dev->rawdev = rawdev;
 	dev->dev_id = dev_id;
+	for (i = 0; i < IFPGA_MAX_IRQ; i++)
+		dev->intr_handle[i] = NULL;
 	dev->poll_enabled = 0;
 
 	return dev;
@@ -1362,36 +1361,62 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 }
 
 int
-ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+ifpga_unregister_msix_irq(struct ifpga_rawdev *dev, enum ifpga_irq_type type,
 		int vec_start, rte_intr_callback_fn handler, void *arg)
 {
-	struct rte_intr_handle *intr_handle;
+	struct rte_intr_handle **intr_handle;
+	int rc = 0;
+	int i = vec_start + 1;
+
+	if (!dev)
+		return -ENODEV;
 
 	if (type == IFPGA_FME_IRQ)
-		intr_handle = &ifpga_irq_handle[0];
+		intr_handle = (struct rte_intr_handle **)&dev->intr_handle[0];
 	else if (type == IFPGA_AFU_IRQ)
-		intr_handle = &ifpga_irq_handle[vec_start + 1];
+		intr_handle = (struct rte_intr_handle **)&dev->intr_handle[i];
 	else
-		return 0;
+		return -EINVAL;
 
-	rte_intr_efd_disable(intr_handle);
+	if ((*intr_handle) == NULL) {
+		IFPGA_RAWDEV_PMD_ERR("%s interrupt %d not registered\n",
+			type == IFPGA_FME_IRQ ? "FME" : "AFU",
+			type == IFPGA_FME_IRQ ? 0 : vec_start);
+		return -ENOENT;
+	}
+
+	rte_intr_efd_disable(*intr_handle);
+
+	rc = rte_intr_callback_unregister(*intr_handle, handler, arg);
+	if (rc < 0) {
+		IFPGA_RAWDEV_PMD_ERR("Failed to unregister %s interrupt %d\n",
+			type == IFPGA_FME_IRQ ? "FME" : "AFU",
+			type == IFPGA_FME_IRQ ? 0 : vec_start);
+	} else {
+		rte_free(*intr_handle);
+		*intr_handle = NULL;
+	}
 
-	return rte_intr_callback_unregister(intr_handle, handler, arg);
+	return rc;
 }
 
 int
-ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+ifpga_register_msix_irq(struct ifpga_rawdev *dev, int port_id,
 		enum ifpga_irq_type type, int vec_start, int count,
 		rte_intr_callback_fn handler, const char *name,
 		void *arg)
 {
 	int ret;
-	struct rte_intr_handle *intr_handle;
+	struct rte_intr_handle **intr_handle;
 	struct opae_adapter *adapter;
 	struct opae_manager *mgr;
 	struct opae_accelerator *acc;
+	int i = 0;
 
-	adapter = ifpga_rawdev_get_priv(dev);
+	if (!dev || !dev->rawdev)
+		return -ENODEV;
+
+	adapter = ifpga_rawdev_get_priv(dev->rawdev);
 	if (!adapter)
 		return -ENODEV;
 
@@ -1400,29 +1425,37 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 		return -ENODEV;
 
 	if (type == IFPGA_FME_IRQ) {
-		intr_handle = &ifpga_irq_handle[0];
+		intr_handle = (struct rte_intr_handle **)&dev->intr_handle[0];
 		count = 1;
 	} else if (type == IFPGA_AFU_IRQ) {
-		intr_handle = &ifpga_irq_handle[vec_start + 1];
+		i = vec_start + 1;
+		intr_handle = (struct rte_intr_handle **)&dev->intr_handle[i];
 	} else {
 		return -EINVAL;
 	}
 
-	intr_handle->type = RTE_INTR_HANDLE_VFIO_MSIX;
+	if (*intr_handle)
+		return -EBUSY;
+
+	*intr_handle = rte_zmalloc(NULL, sizeof(struct rte_intr_handle), 0);
+	if (!(*intr_handle))
+		return -ENOMEM;
+
+	(*intr_handle)->type = RTE_INTR_HANDLE_VFIO_MSIX;
 
-	ret = rte_intr_efd_enable(intr_handle, count);
+	ret = rte_intr_efd_enable(*intr_handle, count);
 	if (ret)
 		return -ENODEV;
 
-	intr_handle->fd = intr_handle->efds[0];
+	(*intr_handle)->fd = (*intr_handle)->efds[0];
 
 	IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
-			name, intr_handle->vfio_dev_fd,
-			intr_handle->fd);
+			name, (*intr_handle)->vfio_dev_fd,
+			(*intr_handle)->fd);
 
 	if (type == IFPGA_FME_IRQ) {
 		struct fpga_fme_err_irq_set err_irq_set;
-		err_irq_set.evtfd = intr_handle->efds[0];
+		err_irq_set.evtfd = (*intr_handle)->efds[0];
 
 		ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
 		if (ret)
@@ -1433,13 +1466,13 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 			return -EINVAL;
 
 		ret = opae_acc_set_irq(acc, vec_start, count,
-				intr_handle->efds);
+				(*intr_handle)->efds);
 		if (ret)
 			return -EINVAL;
 	}
 
 	/* register interrupt handler using DPDK API */
-	ret = rte_intr_callback_register(intr_handle,
+	ret = rte_intr_callback_register(*intr_handle,
 			handler, (void *)arg);
 	if (ret)
 		return -EINVAL;
@@ -1538,7 +1571,7 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 		IFPGA_RAWDEV_PMD_INFO("this is a PF function");
 	}
 
-	ret = ifpga_register_msix_irq(rawdev, 0, IFPGA_FME_IRQ, 0, 0,
+	ret = ifpga_register_msix_irq(dev, 0, IFPGA_FME_IRQ, 0, 0,
 			fme_interrupt_handler, "fme_irq", mgr);
 	if (ret)
 		goto free_adapter_data;
@@ -1588,8 +1621,6 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 		return -EINVAL;
 	}
 	dev = ifpga_rawdev_get(rawdev);
-	if (dev)
-		dev->rawdev = NULL;
 
 	adapter = ifpga_rawdev_get_priv(rawdev);
 	if (!adapter)
@@ -1599,7 +1630,7 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	if (!mgr)
 		return -ENODEV;
 
-	if (ifpga_unregister_msix_irq(IFPGA_FME_IRQ, 0,
+	if (ifpga_unregister_msix_irq(dev, IFPGA_FME_IRQ, 0,
 				fme_interrupt_handler, mgr) < 0)
 		return -EINVAL;
 
@@ -1608,6 +1639,9 @@ static int fme_clean_fme_error(struct opae_manager *mgr)
 	if (ret)
 		IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
 
+	if (dev)
+		dev->rawdev = NULL;
+
 	return ret;
 }
 
diff --git a/drivers/raw/ifpga/ifpga_rawdev.h b/drivers/raw/ifpga/ifpga_rawdev.h
index f082280..b3c7295 100644
--- a/drivers/raw/ifpga/ifpga_rawdev.h
+++ b/drivers/raw/ifpga/ifpga_rawdev.h
@@ -50,6 +50,7 @@ enum ifpga_rawdev_device_state {
 
 #define IFPGA_RAWDEV_MSIX_IRQ_NUM 7
 #define IFPGA_RAWDEV_NUM 32
+#define IFPGA_MAX_IRQ 12
 
 struct ifpga_rawdev {
 	int dev_id;
@@ -59,6 +60,8 @@ struct ifpga_rawdev {
 	uint32_t aer_old[2];
 	char fvl_bdf[8][16];
 	char parent_bdf[16];
+	/* 0 for FME interrupt, others are reserved for AFU irq */
+	void *intr_handle[IFPGA_MAX_IRQ];
 	/* enable monitor thread poll device's sensors or not */
 	int poll_enabled;
 };
@@ -72,12 +75,12 @@ enum ifpga_irq_type {
 };
 
 int
-ifpga_register_msix_irq(struct rte_rawdev *dev, int port_id,
+ifpga_register_msix_irq(struct ifpga_rawdev *dev, int port_id,
 		enum ifpga_irq_type type, int vec_start, int count,
 		rte_intr_callback_fn handler, const char *name,
 		void *arg);
 int
-ifpga_unregister_msix_irq(enum ifpga_irq_type type,
+ifpga_unregister_msix_irq(struct ifpga_rawdev *dev, enum ifpga_irq_type type,
 		int vec_start, rte_intr_callback_fn handler, void *arg);
 
 #endif /* _IFPGA_RAWDEV_H_ */
-- 
1.8.3.1

Re: [PATCH 20.11] raw/ifpga: fix interrupt handle allocation

[Luca Boccassi]

On Fri, 2022-03-11 at 03:27 -0500, Wei Huang wrote:
> [ upstream commit 20659eb38099033b9aa4f7476a7ba7b5d3258569 ]
> 
> Allocate FPGA interrupt handle instance for each card.
> 
> Fixes: e0a1aaf ("raw/ifpga: introduce IRQ functions")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Wei Huang <wei.huang@intel.com>
> Acked-by: Tianfei Zhang <tianfei.zhang@intel.com>
> ---
>  drivers/raw/ifpga/ifpga_rawdev.c | 90 +++++++++++++++++++++++++++-------------
>  drivers/raw/ifpga/ifpga_rawdev.h |  7 +++-
>  2 files changed, 67 insertions(+), 30 deletions(-)

Thanks, applied and pushed.

-- 
Kind regards,
Luca Boccassi