DPDK patches and discussions
 help / color / mirror / Atom feed
From: Haiyue Wang <haiyue.wang@intel.com>
To: dev@dpdk.org, thomas@monjalon.net, vattunuru@marvell.com,
	jerinj@marvell.com, alex.williamson@redhat.com,
	david.marchand@redhat.com
Cc: Haiyue Wang <haiyue.wang@intel.com>
Subject: [dpdk-dev] [PATCH v2] eal: add VFIO-PCI SR-IOV support
Date: Mon, 13 Apr 2020 12:45:14 +0800	[thread overview]
Message-ID: <20200413044514.48266-1-haiyue.wang@intel.com> (raw)
In-Reply-To: <20200305043311.17065-1-vattunuru@marvell.com>

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:
  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:
   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
---
v2: Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
         Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
	   Based on Vamsi's RFC v1, and Alex's patch for Qemu
	       [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]:
	   Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 4 files changed, 80 insertions(+), 7 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..7f99337c7 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e6..a92584795 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..1f9e22d82 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..8c0ad04e5 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.0


  parent reply	other threads:[~2020-04-13  4:53 UTC|newest]

Thread overview: 92+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-03-05  4:33 [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access vattunuru
2020-04-01  3:14 ` Varghese, Vipin
2020-04-10  7:10   ` Vamsi Krishna Attunuru
2020-04-02 12:43 ` Wang, Haiyue
2020-04-10  7:28   ` Vamsi Krishna Attunuru
2020-04-10  7:43     ` Wang, Haiyue
2020-04-10  7:32 ` [dpdk-dev] [RFC v2] eal: add VFIO-PCI SR-IOV support Haiyue Wang
2020-04-10 13:02   ` [dpdk-dev] [EXT] " Vamsi Krishna Attunuru
2020-04-13  2:31 ` [dpdk-dev] [PATCH v1] " Haiyue Wang
2020-04-13  4:45 ` Haiyue Wang [this message]
2020-04-13  6:42 ` [dpdk-dev] [PATCH v3] " Haiyue Wang
2020-04-13  8:29 ` [dpdk-dev] [PATCH v4] " Haiyue Wang
2020-04-13 12:18   ` Thomas Monjalon
2020-04-13 17:01     ` Wang, Haiyue
2020-04-13 15:37   ` Andrew Rybchenko
2020-04-13 16:45     ` Wang, Haiyue
2020-04-14  3:06 ` [dpdk-dev] [PATCH v5 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-04-14  3:06   ` [dpdk-dev] [PATCH v5 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-14  3:06   ` [dpdk-dev] [PATCH v5 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-14  3:21 ` [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-04-14  3:21   ` [dpdk-dev] [PATCH v6 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-14  3:21   ` [dpdk-dev] [PATCH v6 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-14 13:18   ` [dpdk-dev] [EXT] [PATCH v6 0/2] support for VFIO-PCI VF token interface Vamsi Krishna Attunuru
2020-04-18  9:22   ` [dpdk-dev] " David Marchand
2020-04-18  9:38     ` Wang, Haiyue
2020-04-18  9:50       ` Thomas Monjalon
2020-04-18  9:58         ` Wang, Haiyue
2020-04-18 11:16 ` [dpdk-dev] [PATCH v7 " Haiyue Wang
2020-04-18 11:16   ` [dpdk-dev] [PATCH v7 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-18 11:16   ` [dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-18 13:26     ` Thomas Monjalon
2020-04-18 17:37       ` Wang, Haiyue
2020-04-18 17:30 ` [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-04-18 17:30   ` [dpdk-dev] [PATCH v8 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-18 17:30   ` [dpdk-dev] [PATCH v8 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-20 17:12     ` Thomas Monjalon
2020-04-20 16:53   ` [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface David Marchand
2020-04-20 17:02     ` Wang, Haiyue
2020-04-20 17:13       ` Thomas Monjalon
2020-04-20 17:37         ` Wang, Haiyue
2020-04-20 17:42           ` Thomas Monjalon
2020-04-21  1:38             ` Wang, Haiyue
2020-04-21  2:12               ` Thomas Monjalon
2020-04-21  2:52                 ` Wang, Haiyue
2020-04-21  8:47                   ` Thomas Monjalon
2020-04-21 17:35                     ` Wang, Haiyue
2020-04-22  5:08 ` [dpdk-dev] [PATCH v9 " Haiyue Wang
2020-04-22  5:08   ` [dpdk-dev] [PATCH v9 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-22  5:08   ` [dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-22  8:23     ` Thomas Monjalon
2020-04-22 14:02       ` Wang, Haiyue
2020-04-26  1:55 ` [dpdk-dev] [PATCH v10 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-04-26  1:55   ` [dpdk-dev] [PATCH v10 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-04-26  1:55   ` [dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-04-27 11:18     ` Burakov, Anatoly
2020-05-06 16:40     ` Andrew Rybchenko
2020-05-05  7:46 ` [dpdk-dev] [PATCH v11 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-05  7:46   ` [dpdk-dev] [PATCH v11 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-05  7:46   ` [dpdk-dev] [PATCH v11 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-05-05 10:34 ` [dpdk-dev] [PATCH v12 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-05 10:34   ` [dpdk-dev] [PATCH v12 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-05 10:34   ` [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-05-06 10:27     ` Burakov, Anatoly
2020-05-06 11:35       ` Wang, Haiyue
2020-05-06 11:39         ` Burakov, Anatoly
2020-05-06 11:44           ` Wang, Haiyue
2020-05-06 11:35 ` [dpdk-dev] [PATCH v13 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-06 11:35   ` [dpdk-dev] [PATCH v13 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-06 11:35   ` [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-05-06 16:51     ` Andrew Rybchenko
2020-05-06 16:56       ` Wang, Haiyue
2020-05-06 16:58         ` Andrew Rybchenko
2020-05-06 17:06           ` Wang, Haiyue
2020-05-28  1:22 ` [dpdk-dev] [PATCH v14 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-28  1:22   ` [dpdk-dev] [PATCH v14 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-28  1:22   ` [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-05-29  1:26     ` Ye Xiaolong
2020-05-29  1:38       ` Wang, Haiyue
2020-05-29  1:37 ` [dpdk-dev] [PATCH v15 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-05-29  1:37   ` [dpdk-dev] [PATCH v15 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-05-29  1:37   ` [dpdk-dev] [PATCH v15 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-06-17  6:33 ` [dpdk-dev] [PATCH v16 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-06-17  6:33   ` [dpdk-dev] [PATCH v16 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-06-17  6:33   ` [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-06-22 20:39     ` Harman Kalra
2020-06-25  7:33       ` David Marchand
2020-06-25 10:49         ` Wang, Haiyue
2020-07-03 14:57 ` [dpdk-dev] [PATCH v17 0/2] support for VFIO-PCI VF token interface Haiyue Wang
2020-07-03 14:57   ` [dpdk-dev] [PATCH v17 1/2] eal: add uuid dependent header files explicitly Haiyue Wang
2020-07-06 14:32     ` David Marchand
2020-07-03 14:57   ` [dpdk-dev] [PATCH v17 2/2] eal: support for VFIO-PCI VF token Haiyue Wang
2020-07-07 12:05     ` David Marchand

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200413044514.48266-1-haiyue.wang@intel.com \
    --to=haiyue.wang@intel.com \
    --cc=alex.williamson@redhat.com \
    --cc=david.marchand@redhat.com \
    --cc=dev@dpdk.org \
    --cc=jerinj@marvell.com \
    --cc=thomas@monjalon.net \
    --cc=vattunuru@marvell.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).