From: David Marchand <david.marchand@redhat.com>
To: Nipun Gupta <nipun.gupta@amd.com>
Cc: dev@dpdk.org, thomas@monjalon.net, hkalra@marvell.com,
anatoly.burakov@intel.com, stephen@networkplumber.org,
ferruh.yigit@amd.com, harpreet.anand@amd.com,
nikhil.agarwal@amd.com
Subject: Re: [PATCH v5 4/5] bus/cdx: add support for MSI
Date: Thu, 1 Jun 2023 17:09:48 +0200 [thread overview]
Message-ID: <CAJFAV8wi9G5x6oPS-PyFdrmNajtnd_eO71M3B84JD8q8NXw0qw@mail.gmail.com> (raw)
In-Reply-To: <20230525100821.12148-5-nipun.gupta@amd.com>
On Thu, May 25, 2023 at 12:09 PM Nipun Gupta <nipun.gupta@amd.com> wrote:
>
> MSI's are exposed to the devices using VFIO (vfio-cdx). This
> patch uses the same to add support for MSI for the devices on
> the cdx bus.
>
> A couple of API's have been introduced in the EAL interrupt
> framework:
> - rte_intr_irq_count_set: This API is used to set the total
> interrupts on the interrupt handle. This would be provided
> by VFIO (irq.count) for VFIO enabled devices.
> - rte_intr_irq_count_get: This API returns the total number
> interrupts which were set.
>
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> Acked-by: Ferruh Yigit <ferruh.yigit@amd.com>
> ---
> drivers/bus/cdx/bus_cdx_driver.h | 25 +++++
> drivers/bus/cdx/cdx.c | 11 ++
> drivers/bus/cdx/cdx_vfio.c | 182 ++++++++++++++++++++++++++++++-
> drivers/bus/cdx/version.map | 2 +
> 4 files changed, 218 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/bus/cdx/bus_cdx_driver.h b/drivers/bus/cdx/bus_cdx_driver.h
> index f1dce06a16..a8c54d728e 100644
> --- a/drivers/bus/cdx/bus_cdx_driver.h
> +++ b/drivers/bus/cdx/bus_cdx_driver.h
> @@ -67,6 +67,7 @@ struct rte_cdx_device {
> struct rte_cdx_id id; /**< CDX ID. */
> struct rte_mem_resource mem_resource[CDX_MAX_RESOURCE];
> /**< CDX Memory Resource */
> + struct rte_intr_handle *intr_handle; /**< Interrupt handle */
> };
>
> /**
> @@ -168,6 +169,30 @@ void rte_cdx_unmap_device(struct rte_cdx_device *dev);
> __rte_internal
> void rte_cdx_register(struct rte_cdx_driver *driver);
>
> +/**
> + * Enables VFIO Interrupts for CDX bus devices.
> + *
> + * @param intr_handle
> + * Pointer to the interrupt handle.
> + *
> + * @return
> + * 0 on success, -1 on error.
> + */
> +__rte_internal
> +int rte_cdx_vfio_intr_enable(const struct rte_intr_handle *intr_handle);
> +
> +/**
> + * Disable VFIO Interrupts for CDX bus devices.
> + *
> + * @param intr_handle
> + * Pointer to the interrupt handle.
> + *
> + * @return
> + * 0 on success, -1 on error.
> + */
> +__rte_internal
> +int rte_cdx_vfio_intr_disable(const struct rte_intr_handle *intr_handle);
> +
> /**
> * Helper for CDX device registration from driver (eth, crypto, raw) instance
> */
> diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
> index 64ea879f3b..c691c38e04 100644
> --- a/drivers/bus/cdx/cdx.c
> +++ b/drivers/bus/cdx/cdx.c
> @@ -201,6 +201,15 @@ cdx_scan_one(const char *dirname, const char *dev_name)
> goto err;
> }
>
> + /* Allocate interrupt instance for cdx device */
> + dev->intr_handle =
> + rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE);
> + if (dev->intr_handle == NULL) {
> + CDX_BUS_ERR("Failed to create interrupt instance for %s\n",
CDX_BUS_ERR already appends a \n.
> + dev->device.name);
> + return -ENOMEM;
> + }
> +
> /*
> * Check if device is bound to 'vfio-cdx' driver, so that user-space
> * can gracefully access the device.
> @@ -391,6 +400,8 @@ cdx_probe_one_driver(struct rte_cdx_driver *dr,
> return ret;
>
> error_probe:
> + rte_intr_instance_free(dev->intr_handle);
> + dev->intr_handle = NULL;
> cdx_vfio_unmap_resource(dev);
> error_map_device:
> return ret;
> diff --git a/drivers/bus/cdx/cdx_vfio.c b/drivers/bus/cdx/cdx_vfio.c
> index e54432de5b..ec6512e158 100644
> --- a/drivers/bus/cdx/cdx_vfio.c
> +++ b/drivers/bus/cdx/cdx_vfio.c
> @@ -50,6 +50,10 @@ struct mapped_cdx_resource {
> /** mapped cdx device list */
> TAILQ_HEAD(mapped_cdx_res_list, mapped_cdx_resource);
>
> +/* IRQ set buffer length for MSI interrupts */
> +#define MSI_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
> + sizeof(int) * (RTE_MAX_RXTX_INTR_VEC_ID + 1))
> +
> static struct rte_tailq_elem cdx_vfio_tailq = {
> .name = "VFIO_CDX_RESOURCE_LIST",
> };
> @@ -94,6 +98,27 @@ cdx_vfio_unmap_resource_primary(struct rte_cdx_device *dev)
> char cdx_addr[PATH_MAX] = {0};
> struct mapped_cdx_resource *vfio_res = NULL;
> struct mapped_cdx_res_list *vfio_res_list;
> + int ret, vfio_dev_fd;
> +
> + if (rte_intr_fd_get(dev->intr_handle) < 0)
> + return -1;
> +
> + if (close(rte_intr_fd_get(dev->intr_handle)) < 0) {
> + CDX_BUS_ERR("Error when closing eventfd file descriptor for %s",
> + dev->device.name);
> + return -1;
> + }
> +
> + vfio_dev_fd = rte_intr_dev_fd_get(dev->intr_handle);
> + if (vfio_dev_fd < 0)
> + return -1;
> +
> + ret = rte_vfio_release_device(rte_cdx_get_sysfs_path(), dev->device.name,
> + vfio_dev_fd);
> + if (ret < 0) {
> + CDX_BUS_ERR("Cannot release VFIO device");
> + return ret;
> + }
>
> vfio_res_list =
> RTE_TAILQ_CAST(cdx_vfio_tailq.head, mapped_cdx_res_list);
> @@ -116,6 +141,18 @@ cdx_vfio_unmap_resource_secondary(struct rte_cdx_device *dev)
> {
> struct mapped_cdx_resource *vfio_res = NULL;
> struct mapped_cdx_res_list *vfio_res_list;
> + int ret, vfio_dev_fd;
> +
> + vfio_dev_fd = rte_intr_dev_fd_get(dev->intr_handle);
> + if (vfio_dev_fd < 0)
> + return -1;
> +
> + ret = rte_vfio_release_device(rte_cdx_get_sysfs_path(), dev->device.name,
> + vfio_dev_fd);
> + if (ret < 0) {
> + CDX_BUS_ERR("Cannot release VFIO device");
> + return ret;
> + }
>
> vfio_res_list =
> RTE_TAILQ_CAST(cdx_vfio_tailq.head, mapped_cdx_res_list);
> @@ -140,9 +177,80 @@ cdx_vfio_unmap_resource(struct rte_cdx_device *dev)
> return cdx_vfio_unmap_resource_secondary(dev);
> }
>
> +/* set up interrupt support (but not enable interrupts) */
> static int
> -cdx_rte_vfio_setup_device(int vfio_dev_fd)
> +cdx_vfio_setup_interrupts(struct rte_cdx_device *dev, int vfio_dev_fd,
Why rename this function?
> + int num_irqs)
> {
> + int i, ret;
> +
> + if (num_irqs == 0)
> + return 0;
> +
> + /* start from MSI interrupt type */
> + for (i = 0; i < num_irqs; i++) {
[snip]
--
David Marchand
next prev parent reply other threads:[~2023-06-01 15:10 UTC|newest]
Thread overview: 102+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-01-24 14:07 [RFC PATCH 0/6] add support for CDX bus Nipun Gupta
2023-01-24 14:07 ` [RFC PATCH 1/6] bus/cdx: introduce cdx bus Nipun Gupta
2023-01-24 14:07 ` [RFC PATCH 2/6] bus/cdx: add dma map and unmap support Nipun Gupta
2023-01-24 14:07 ` [RFC PATCH 3/6] bus/cdx: add support for MSI Nipun Gupta
2023-01-24 14:07 ` [RFC PATCH 4/6] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-01-24 14:07 ` [RFC PATCH 5/6] bus: enable cdx bus Nipun Gupta
2023-01-24 14:07 ` [RFC PATCH 6/6] config/arm: add AMD CDX Nipun Gupta
2023-04-07 6:01 ` [PATCH 0/6] add support for CDX bus Nipun Gupta
2023-04-07 6:01 ` [PATCH 1/6] bus/cdx: introduce cdx bus Nipun Gupta
2023-04-07 6:01 ` [PATCH 2/6] bus/cdx: add dma map and unmap support Nipun Gupta
2023-04-07 6:01 ` [PATCH 3/6] bus/cdx: add support for MSI Nipun Gupta
2023-04-07 6:01 ` [PATCH 4/6] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-04-07 6:01 ` [PATCH 5/6] bus: enable cdx bus Nipun Gupta
2023-04-07 6:01 ` [PATCH 6/6] config/arm: add AMD CDX Nipun Gupta
2023-04-07 7:18 ` [PATCH 0/6] add support for CDX bus David Marchand
2023-04-07 7:29 ` Nipun Gupta
2023-04-13 13:25 ` Gupta, Nipun
2023-04-13 13:26 ` [PATCH v2 " Nipun Gupta
2023-04-13 13:26 ` [PATCH v2 1/6] bus/cdx: introduce cdx bus Nipun Gupta
2023-04-14 16:45 ` Ferruh Yigit
2023-04-16 9:20 ` Gupta, Nipun
2023-04-13 13:27 ` [PATCH v2 2/6] bus/cdx: add dma map and unmap support Nipun Gupta
2023-04-13 13:27 ` [PATCH v2 3/6] bus/cdx: add support for MSI Nipun Gupta
2023-04-14 16:45 ` Ferruh Yigit
2023-04-16 9:20 ` Gupta, Nipun
2023-04-13 13:27 ` [PATCH v2 4/6] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-04-13 13:27 ` [PATCH v2 5/6] bus: enable cdx bus Nipun Gupta
2023-04-14 16:45 ` Ferruh Yigit
2023-04-16 9:21 ` Gupta, Nipun
2023-04-13 13:27 ` [PATCH v2 6/6] config/arm: add AMD CDX Nipun Gupta
2023-04-14 16:45 ` [PATCH v2 0/6] add support for CDX bus Ferruh Yigit
2023-04-21 14:54 ` [PATCH v3 0/5] Support AMD CDX bus, for FPGA based CDX devices. The CDX Nipun Gupta
2023-04-21 14:54 ` [PATCH v3 1/5] bus/cdx: introduce cdx bus Nipun Gupta
2023-04-21 14:54 ` [PATCH v3 2/5] bus/cdx: add DMA map and unmap support Nipun Gupta
2023-04-21 14:54 ` [PATCH v3 3/5] bus/cdx: add support for MSI Nipun Gupta
2023-04-21 14:54 ` [PATCH v3 4/5] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-04-21 14:54 ` [PATCH v3 5/5] config/arm: add AMD CDX Nipun Gupta
2023-05-04 15:28 ` Ferruh Yigit
2023-05-08 10:24 ` Gupta, Nipun
2023-05-08 10:44 ` Thomas Monjalon
2023-05-08 10:48 ` Gupta, Nipun
2023-05-08 11:26 ` Ferruh Yigit
2023-05-08 17:16 ` Honnappa Nagarahalli
2023-05-08 17:47 ` Ferruh Yigit
2023-05-09 4:35 ` Gupta, Nipun
2023-05-09 5:55 ` Ruifeng Wang
2023-06-14 10:30 ` Ferruh Yigit
2023-06-15 7:00 ` Ruifeng Wang
2023-05-08 11:18 ` [PATCH v4 0/4] Support AMD CDX bus, for FPGA based CDX devices. The CDX Nipun Gupta
2023-05-08 11:18 ` [PATCH v4 1/4] bus/cdx: introduce cdx bus Nipun Gupta
2023-05-09 6:54 ` Xia, Chenbo
2023-05-09 11:09 ` Gupta, Nipun
2023-05-09 11:25 ` Ferruh Yigit
2023-05-10 1:29 ` Xia, Chenbo
2023-05-24 11:14 ` Thomas Monjalon
2023-05-24 17:04 ` Gupta, Nipun
2023-05-08 11:18 ` [PATCH v4 2/4] bus/cdx: add DMA map and unmap support Nipun Gupta
2023-05-08 11:18 ` [PATCH v4 3/4] bus/cdx: add support for MSI Nipun Gupta
2023-05-24 11:06 ` Thomas Monjalon
2023-05-24 17:06 ` Gupta, Nipun
2023-05-08 11:18 ` [PATCH v4 4/4] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-05-12 10:25 ` [PATCH v4 0/4] Support AMD CDX bus, for FPGA based CDX devices. The CDX Ferruh Yigit
2023-05-25 10:08 ` [PATCH v5 0/5] Support AMD CDX bus Nipun Gupta
2023-05-25 10:08 ` [PATCH v5 1/5] bus/cdx: introduce " Nipun Gupta
2023-06-01 15:00 ` David Marchand
2023-06-01 18:10 ` Nipun Gupta
2023-06-05 8:04 ` Nipun Gupta
2023-05-25 10:08 ` [PATCH v5 2/5] bus/cdx: add DMA map and unmap support Nipun Gupta
2023-06-01 15:07 ` David Marchand
2023-05-25 10:08 ` [PATCH v5 3/5] eal/interrupts: add IRQ count in interrupt handle Nipun Gupta
2023-06-01 15:25 ` David Marchand
2023-06-01 18:04 ` Nipun Gupta
2023-06-01 18:18 ` Gupta, Nipun
2023-06-06 7:18 ` [EXT] " Harman Kalra
2023-06-06 7:27 ` Nipun Gupta
2023-06-07 5:45 ` Harman Kalra
2023-05-25 10:08 ` [PATCH v5 4/5] bus/cdx: add support for MSI Nipun Gupta
2023-06-01 15:09 ` David Marchand [this message]
2023-06-05 8:05 ` Nipun Gupta
2023-05-25 10:08 ` [PATCH v5 5/5] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-06-05 13:26 ` [PATCH v6 0/4] Support AMD CDX bus Nipun Gupta
2023-06-05 13:26 ` [PATCH v6 1/4] bus/cdx: introduce " Nipun Gupta
2023-06-06 8:55 ` Thomas Monjalon
2023-06-05 13:26 ` [PATCH v6 2/4] bus/cdx: add DMA map and unmap support Nipun Gupta
2023-06-05 13:26 ` [PATCH v6 3/4] bus/cdx: add support for MSI Nipun Gupta
2023-06-06 9:30 ` David Marchand
2023-06-06 9:42 ` Nipun Gupta
2023-06-05 13:26 ` [PATCH v6 4/4] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-06-06 10:02 ` [PATCH v7 0/4] Support AMD CDX bus Nipun Gupta
2023-06-06 10:02 ` [PATCH v7 1/4] bus/cdx: introduce " Nipun Gupta
2023-06-06 13:00 ` Thomas Monjalon
2023-06-06 13:38 ` Nipun Gupta
2023-06-06 13:43 ` Thomas Monjalon
2023-06-06 10:02 ` [PATCH v7 2/4] bus/cdx: add DMA map and unmap support Nipun Gupta
2023-06-06 10:02 ` [PATCH v7 3/4] bus/cdx: add support for MSI Nipun Gupta
2023-06-06 10:02 ` [PATCH v7 4/4] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-06-07 4:24 ` [PATCH v8 0/4] Support AMD CDX bus Nipun Gupta
2023-06-07 4:24 ` [PATCH v8 1/4] bus/cdx: introduce " Nipun Gupta
2023-06-07 4:24 ` [PATCH v8 2/4] bus/cdx: add DMA map and unmap support Nipun Gupta
2023-06-07 4:24 ` [PATCH v8 3/4] bus/cdx: add support for MSI Nipun Gupta
2023-06-07 4:24 ` [PATCH v8 4/4] bus/cdx: support plug unplug and dev iterator Nipun Gupta
2023-06-07 13:36 ` [PATCH v8 0/4] Support AMD CDX bus Thomas Monjalon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=CAJFAV8wi9G5x6oPS-PyFdrmNajtnd_eO71M3B84JD8q8NXw0qw@mail.gmail.com \
--to=david.marchand@redhat.com \
--cc=anatoly.burakov@intel.com \
--cc=dev@dpdk.org \
--cc=ferruh.yigit@amd.com \
--cc=harpreet.anand@amd.com \
--cc=hkalra@marvell.com \
--cc=nikhil.agarwal@amd.com \
--cc=nipun.gupta@amd.com \
--cc=stephen@networkplumber.org \
--cc=thomas@monjalon.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).