From: Vamsi Krishna Attunuru <vattunuru@marvell.com>
To: Haiyue Wang <haiyue.wang@intel.com>,
"dev@dpdk.org" <dev@dpdk.org>,
"thomas@monjalon.net" <thomas@monjalon.net>,
Jerin Jacob Kollanukkaran <jerinj@marvell.com>,
"alex.williamson@redhat.com" <alex.williamson@redhat.com>,
"david.marchand@redhat.com" <david.marchand@redhat.com>
Subject: Re: [dpdk-dev] [EXT] [RFC v2] eal: add VFIO-PCI SR-IOV support
Date: Fri, 10 Apr 2020 13:02:43 +0000 [thread overview]
Message-ID: <MWHPR18MB1645E9D768C2F0804B253DBCA6DE0@MWHPR18MB1645.namprd18.prod.outlook.com> (raw)
In-Reply-To: <20200410073254.34905-1-haiyue.wang@intel.com>
> -----Original Message-----
> From: Haiyue Wang <haiyue.wang@intel.com>
> Sent: Friday, April 10, 2020 1:03 PM
> To: dev@dpdk.org; thomas@monjalon.net; Vamsi Krishna Attunuru
> <vattunuru@marvell.com>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> alex.williamson@redhat.com; david.marchand@redhat.com
> Cc: Haiyue Wang <haiyue.wang@intel.com>
> Subject: [EXT] [RFC v2] eal: add VFIO-PCI SR-IOV support
>
> External Email
>
> ----------------------------------------------------------------------
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support.
>
> The VF token can be set by a vfio-pci based PF driver and must be known by
> the vfio-pci based VF driver in order to gain access to the device.
>
> An example VF token option would take this form:
>
> 1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
>
> 2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
>
> 3. Start the PF:
> ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
> -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
> --file-prefix=pf -- -i
>
> 4. Start the VF:
> ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
> -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
> --file-prefix=vf1 -- -i
>
> Test based on vfio-pci patch:
> https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.ozlabs.org_cover_1253222_&d=DwIDAg&c=nKjWec2b6R0m
> OyPaz7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SFD0o
> TfFgnU88wT2X7qMWRDen3KFV8oSOlqz3WQm3nI&s=CU2jxKUsy4oiI26apob
> DF8BJebDrheBiu3eKlF3e76E&e=
>
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> ---
> Based on RFC v1: https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_patch_66281_&d=DwIDAg&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SFD0oTfFg
> nU88wT2X7qMWRDen3KFV8oSOlqz3WQm3nI&s=3dHnLcXpkRVOZs6wdZQylc
> Je0oNNBs77hXE6SOuLO7o&e=
> ---
> drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++--
> lib/librte_eal/freebsd/eal.c | 3 +-
> lib/librte_eal/include/rte_vfio.h | 8 ++++-
> lib/librte_eal/linux/eal_vfio.c | 20 +++++++++--
> 4 files changed, 80 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
> index 64cd84a68..7f99337c7 100644
> --- a/drivers/bus/pci/linux/pci_vfio.c
> +++ b/drivers/bus/pci/linux/pci_vfio.c
> @@ -11,6 +11,7 @@
> #include <sys/mman.h>
> #include <stdbool.h>
>
> +#include <rte_devargs.h>
> #include <rte_log.h>
> #include <rte_pci.h>
> #include <rte_bus_pci.h>
> @@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int
> msix_region)
> return ret;
> }
>
> +static void
> +vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) {
> +#define VF_TOKEN_ARG "vf_token="
> + char c, *p, *vf_token;
> +
> + if (devargs == NULL)
> + return;
> +
> + p = strstr(devargs->args, VF_TOKEN_ARG);
> + if (!p)
> + return;
> +
> + vf_token = p + strlen(VF_TOKEN_ARG);
> + if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
> + return;
> +
> + c = vf_token[RTE_UUID_STRLEN - 1];
> + if (c != '\0' && c != ',')
> + return;
> +
> + vf_token[RTE_UUID_STRLEN - 1] = '\0';
> + if (rte_uuid_parse(vf_token, uu)) {
> + RTE_LOG(ERR, EAL,
> + "The VF token is not a valid uuid : %s\n", vf_token);
> + vf_token[RTE_UUID_STRLEN - 1] = c;
> + return;
> + }
> +
> + RTE_LOG(DEBUG, EAL,
> + "The VF token is found : %s\n", vf_token);
> +
> + vf_token[RTE_UUID_STRLEN - 1] = c;
> +
> + /* Purge this vfio-pci specific token from the device arguments */
> + if (c != '\0') {
> + /* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
> + memmove(p, vf_token + RTE_UUID_STRLEN,
> + strlen(vf_token + RTE_UUID_STRLEN) + 1);
> + } else {
> + /* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
> + if (p != devargs->args)
> + p--;
> +
> + *p = '\0';
> + }
> +}
>
> static int
> pci_vfio_map_resource_primary(struct rte_pci_device *dev) {
> struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> + rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
> char pci_addr[PATH_MAX] = {0};
> int vfio_dev_fd;
> struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +717,9 @@
> pci_vfio_map_resource_primary(struct rte_pci_device *dev)
> snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
> loc->domain, loc->bus, loc->devid, loc->function);
>
> + vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
> ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> - &vfio_dev_fd, &device_info);
> + &vfio_dev_fd, &device_info,
> vf_token);
> if (ret)
> return ret;
>
> @@ -797,6 +847,7 @@ static int
> pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {
> struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> + rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
> char pci_addr[PATH_MAX] = {0};
> int vfio_dev_fd;
> struct rte_pci_addr *loc = &dev->addr; @@ -830,8 +881,9 @@
> pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
> return -1;
> }
>
> + vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
> ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> - &vfio_dev_fd, &device_info);
> + &vfio_dev_fd, &device_info,
> vf_token);
> if (ret)
> return ret;
>
> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c index
> 6ae37e7e6..cbb4c400e 100644
> --- a/lib/librte_eal/freebsd/eal.c
> +++ b/lib/librte_eal/freebsd/eal.c
> @@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int
> rte_vfio_setup_device(__rte_unused const char *sysfs_base,
> __rte_unused const char *dev_addr,
> __rte_unused int *vfio_dev_fd,
> - __rte_unused struct vfio_device_info *device_info)
> + __rte_unused struct vfio_device_info *device_info
> + __rte_unused rte_uuid_t vf_token)
> {
> return -1;
> }
> diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
> index 20ed8c45a..1f9e22d82 100644
> --- a/lib/librte_eal/include/rte_vfio.h
> +++ b/lib/librte_eal/include/rte_vfio.h
> @@ -16,6 +16,8 @@ extern "C" {
>
> #include <stdint.h>
>
> +#include <rte_uuid.h>
> +
> /*
> * determine if VFIO is present on the system
> */
> @@ -102,13 +104,17 @@ struct vfio_device_info;
> * @param device_info
> * Device information.
> *
> + * @param vf_token
> + * VF token.
> + *
> * @return
> * 0 on success.
> * <0 on failure.
> * >1 if the device cannot be managed this way.
> */
> int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> - int *vfio_dev_fd, struct vfio_device_info *device_info);
> + int *vfio_dev_fd, struct vfio_device_info *device_info,
> + rte_uuid_t vf_token);
>
> /**
> * Release a device mapped to a VFIO-managed I/O MMU group.
> diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
> index 4502aefed..8c0ad04e5 100644
> --- a/lib/librte_eal/linux/eal_vfio.c
> +++ b/lib/librte_eal/linux/eal_vfio.c
> @@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
>
> int
> rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> - int *vfio_dev_fd, struct vfio_device_info *device_info)
> + int *vfio_dev_fd, struct vfio_device_info *device_info,
> + rte_uuid_t vf_token)
> {
> struct vfio_group_status group_status = {
> .argsz = sizeof(group_status)
> @@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const
> char *dev_addr,
> int vfio_container_fd;
> int vfio_group_fd;
> int iommu_group_num;
> + char dev[PATH_MAX];
> int i, ret;
>
> /* get group number */
> @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,
> const char *dev_addr,
> t->type_id, t->name);
> }
>
> + if (!rte_uuid_is_null(vf_token)) {
> + char vf_token_str[RTE_UUID_STRLEN];
> +
> + rte_uuid_unparse(vf_token, vf_token_str,
> sizeof(vf_token_str));
> + snprintf(dev, sizeof(dev),
> + "%s vf_token=%s", dev_addr, vf_token_str);
> + } else {
> + snprintf(dev, sizeof(dev),
> + "%s", dev_addr);
> + }
> +
> /* get a file descriptor for the device */
> - *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> dev_addr);
> + *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> dev);
> if (*vfio_dev_fd < 0) {
> /* if we cannot get a device fd, this implies a problem with
> * the VFIO group or the container not having IOMMU
> configured.
> @@ -2081,7 +2094,8 @@ int
> rte_vfio_setup_device(__rte_unused const char *sysfs_base,
> __rte_unused const char *dev_addr,
> __rte_unused int *vfio_dev_fd,
> - __rte_unused struct vfio_device_info *device_info)
> + __rte_unused struct vfio_device_info *device_info
> + __rte_unused rte_uuid_t vf_token)
> {
> return -1;
> }
> --
> 2.26.0
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
next prev parent reply other threads:[~2020-04-10 13:02 UTC|newest]
Thread overview: 92+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-03-05 4:33 [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access vattunuru
2020-04-01 3:14 ` Varghese, Vipin
2020-04-10 7:10 ` Vamsi Krishna Attunuru
2020-04-02 12:43 ` Wang, Haiyue
2020-04-10 7:28 ` Vamsi Krishna Attunuru
2020-04-10 7:43 ` Wang, Haiyue
2020-04-10 7:32 ` [dpdk-dev] [RFC v2] eal: add VFIO-PCI SR-IOV support Haiyue Wang
2020-04-10 13:02 ` Vamsi Krishna Attunuru [this message]
2020-04-13 2:31 ` [dpdk-dev] [PATCH v1] " Haiyue Wang
2020-04-13 4:45 ` [dpdk-dev] [PATCH v2] " Haiyue Wang
2020-04-13 6:42 ` [dpdk-dev] [PATCH v3] " Haiyue Wang
2020-04-13 8:29 ` [dpdk-dev] [PATCH v4] " Haiyue Wang
2020-04-13 12:18 ` Thomas Monjalon
2020-04-13 17:01 ` Wang, Haiyue
2020-04-13 15:37 ` Andrew Rybchenko
2020-04-13 16:45 ` Wang, Haiyue
2020-04-14 3:06 ` [dpdk-dev] [PATCH v5 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-04-14 3:06 ` [dpdk-dev] [PATCH v5 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-14 3:06 ` [dpdk-dev] [PATCH v5 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-14 3:21 ` [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-04-14 3:21 ` [dpdk-dev] [PATCH v6 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-14 3:21 ` [dpdk-dev] [PATCH v6 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-14 13:18 ` [dpdk-dev] [EXT] [PATCH v6 0/2] support for VFIO-PCI VF token interface Vamsi Krishna Attunuru
2020-04-18 9:22 ` [dpdk-dev] " David Marchand
2020-04-18 9:38 ` Wang, Haiyue
2020-04-18 9:50 ` Thomas Monjalon
2020-04-18 9:58 ` Wang, Haiyue
2020-04-18 11:16 ` [dpdk-dev] [PATCH v7 " Haiyue Wang
2020-04-18 11:16 ` [dpdk-dev] [PATCH v7 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-18 11:16 ` [dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-18 13:26 ` Thomas Monjalon
2020-04-18 17:37 ` Wang, Haiyue
2020-04-18 17:30 ` [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-04-18 17:30 ` [dpdk-dev] [PATCH v8 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-18 17:30 ` [dpdk-dev] [PATCH v8 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-20 17:12 ` Thomas Monjalon
2020-04-20 16:53 ` [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface David Marchand
2020-04-20 17:02 ` Wang, Haiyue
2020-04-20 17:13 ` Thomas Monjalon
2020-04-20 17:37 ` Wang, Haiyue
2020-04-20 17:42 ` Thomas Monjalon
2020-04-21 1:38 ` Wang, Haiyue
2020-04-21 2:12 ` Thomas Monjalon
2020-04-21 2:52 ` Wang, Haiyue
2020-04-21 8:47 ` Thomas Monjalon
2020-04-21 17:35 ` Wang, Haiyue
2020-04-22 5:08 ` [dpdk-dev] [PATCH v9 " Haiyue Wang
2020-04-22 5:08 ` [dpdk-dev] [PATCH v9 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-22 5:08 ` [dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-22 8:23 ` Thomas Monjalon
2020-04-22 14:02 ` Wang, Haiyue
2020-04-26 1:55 ` [dpdk-dev] [PATCH v10 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-04-26 1:55 ` [dpdk-dev] [PATCH v10 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-26 1:55 ` [dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-27 11:18 ` Burakov, Anatoly
2020-05-06 16:40 ` Andrew Rybchenko
2020-05-05 7:46 ` [dpdk-dev] [PATCH v11 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-05 7:46 ` [dpdk-dev] [PATCH v11 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-05 7:46 ` [dpdk-dev] [PATCH v11 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-05-05 10:34 ` [dpdk-dev] [PATCH v12 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-05 10:34 ` [dpdk-dev] [PATCH v12 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-05 10:34 ` [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-05-06 10:27 ` Burakov, Anatoly
2020-05-06 11:35 ` Wang, Haiyue
2020-05-06 11:39 ` Burakov, Anatoly
2020-05-06 11:44 ` Wang, Haiyue
2020-05-06 11:35 ` [dpdk-dev] [PATCH v13 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-06 11:35 ` [dpdk-dev] [PATCH v13 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-06 11:35 ` [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-05-06 16:51 ` Andrew Rybchenko
2020-05-06 16:56 ` Wang, Haiyue
2020-05-06 16:58 ` Andrew Rybchenko
2020-05-06 17:06 ` Wang, Haiyue
2020-05-28 1:22 ` [dpdk-dev] [PATCH v14 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-28 1:22 ` [dpdk-dev] [PATCH v14 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-28 1:22 ` [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-05-29 1:26 ` Ye Xiaolong
2020-05-29 1:38 ` Wang, Haiyue
2020-05-29 1:37 ` [dpdk-dev] [PATCH v15 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-29 1:37 ` [dpdk-dev] [PATCH v15 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-29 1:37 ` [dpdk-dev] [PATCH v15 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-06-17 6:33 ` [dpdk-dev] [PATCH v16 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-06-17 6:33 ` [dpdk-dev] [PATCH v16 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-06-17 6:33 ` [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-06-22 20:39 ` Harman Kalra
2020-06-25 7:33 ` David Marchand
2020-06-25 10:49 ` Wang, Haiyue
2020-07-03 14:57 ` [dpdk-dev] [PATCH v17 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-07-03 14:57 ` [dpdk-dev] [PATCH v17 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-07-06 14:32 ` David Marchand
2020-07-03 14:57 ` [dpdk-dev] [PATCH v17 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-07-07 12:05 ` David Marchand
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=MWHPR18MB1645E9D768C2F0804B253DBCA6DE0@MWHPR18MB1645.namprd18.prod.outlook.com \
--to=vattunuru@marvell.com \
--cc=alex.williamson@redhat.com \
--cc=david.marchand@redhat.com \
--cc=dev@dpdk.org \
--cc=haiyue.wang@intel.com \
--cc=jerinj@marvell.com \
--cc=thomas@monjalon.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).