[dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

[dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

[vattunuru]

From: Vamsi Attunuru <vattunuru@marvell.com>

vfio-pci driver enables virtual function access from
the DPDK applications when those vf device's physical
function is also bound to vfio driver.

Patch adds the required configuration and checks to
enable DPDK applications to access both pf and it's
vf devices through vfio-pci driver.

See background on vf token scheme in linux vfio driver.
http://patches.dpdk.org/cover/65915/

When a physical function is enabled with  non-zero
virtual functions,  patch sets the UUID using
VFIO_DEVICE_FEATURE ioctl from physical function's
file descriptor. Same UUID is used to gain the access
for the virtual functions on those  physical function.

Following changes required on top of this DPDK patch
* Kernel version check for VFIO_DEVICE_FEATURE ioctl
* Use uuid gen API to generate UUID.

Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>

diff --git a/lib/librte_eal/linux/eal/eal_vfio.c b/lib/librte_eal/linux/eal/eal_vfio.c
index 01b5ef3..e2fdd35 100644
--- a/lib/librte_eal/linux/eal/eal_vfio.c
+++ b/lib/librte_eal/linux/eal/eal_vfio.c
@@ -12,6 +12,7 @@
 #include <rte_log.h>
 #include <rte_memory.h>
 #include <rte_eal_memconfig.h>
+#include <rte_uuid.h>
 #include <rte_vfio.h>
 
 #include "eal_filesystem.h"
@@ -50,6 +51,9 @@ struct vfio_config {
 	struct user_mem_maps mem_maps;
 };
 
+rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
+				      0x913f, 0xf2d2f965ed0eULL);
+
 /* per-process VFIO config */
 static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];
 static struct vfio_config *default_vfio_cfg = &vfio_cfgs[0];
@@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)
 	return 0;
 }
 
+static bool
+rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	/* check if physfn directory exist for this device */
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+
+	/* For PFs, physfn directory does not exist */
+	if (ret < 0)
+		return true;
+
+	return false;
+}
+
+static int
+is_vf_token_required(const char *sysfs_base, const char *dev_addr)
+{
+	char *tok[16], *physfn, *physfn_drv;
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
+		return -1;
+	}
+
+	physfn = tok[ret - 1];
+
+	snprintf(linkname, sizeof(linkname),
+			 "/sys/bus/pci/devices/%s/driver", physfn);
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
+			dev_addr);
+		return -1;
+	}
+
+	physfn_drv = tok[ret - 1];
+
+	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
+		return 1;
+
+	/* physfn is bound to vfio-pci */
+	return 0;
+}
+
+static bool
+rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	unsigned long num_vfs;
+	int ret;
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
+		return false;
+
+	memset(linkname, 0, sizeof(linkname));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
+
+	ret  = eal_parse_sysfs_value(linkname, &num_vfs);
+
+	if ((ret < 0) || (num_vfs == 0))
+		return false;
+
+	return true;
+}
+
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		int *vfio_dev_fd, struct vfio_device_info *device_info)
@@ -669,6 +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	if (ret < 0)
 		return -1;
 
+	snprintf(dev, sizeof(dev), "%s", dev_addr);
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
+		char vf_token[PATH_MAX];
+		/*
+		 *  Check if vf_token is required or not,
+		 *  vf_token is required when the VF's physfn is
+		 *  binded with vfio-pci driver
+		 */
+		ret = is_vf_token_required(sysfs_base, dev_addr);
+		/* if negative, something failed */
+		if (ret < 0)
+			return -1;
+
+		if (ret == 0) {
+			/* vf_token required to open device file descriptor */
+			rte_uuid_unparse(uuid_token,
+					 vf_token, sizeof(vf_token));
+			snprintf(dev, sizeof(dev),
+				 "%s vf_token=%s", dev_addr, vf_token);
+		}
+	}
+
 	/* get the actual group fd */
 	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
 	if (vfio_group_fd < 0)
@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	}
 
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		rte_vfio_clear_group(vfio_group_fd);
 		return -1;
 	}
+
+	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {
+#define VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
+
+		struct vfio_device_feature *vf_token;
+		uint8_t local[VF_TOKEN];
+
+		memset(local, 0, VF_TOKEN);
+		vf_token = (struct vfio_device_feature *)local;
+		vf_token->argsz = VF_TOKEN;
+		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
+					VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
+
+		memcpy(local + sizeof(struct vfio_device_feature),
+		       &uuid_token, sizeof(uuid_token));
+
+		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
+		if (ret) {
+			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
+					"error %i (%s)\n", dev_addr, errno,
+					strerror(errno));
+			return -1;
+		}
+	}
+
 	vfio_group_device_get(vfio_group_fd);
 
 	return 0;
-- 
2.8.4

Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

[Varghese, Vipin]

Snipped
> diff --git a/lib/librte_eal/linux/eal/eal_vfio.c
> b/lib/librte_eal/linux/eal/eal_vfio.c
> index 01b5ef3..e2fdd35 100644
> --- a/lib/librte_eal/linux/eal/eal_vfio.c
> +++ b/lib/librte_eal/linux/eal/eal_vfio.c

DPDK supports freebsd where `/dev/pci` can be probed like Linux `/sys/bus/pci`. Will you be adding the functions in `lib/librte_eal/freebsd/eal/` folder or implement as `NOT supported`?

What about window ` lib/librte_eal/windows/eal/`?

snipped
> +	snprintf(linkname, sizeof(linkname),
> +			 "%s/%s/physfn", sysfs_base, dev_addr);

Would you like to check the pointer sysfs_base or dev_addr?

>  int
>  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  		int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -

Snipped

> +		ret = is_vf_token_required(sysfs_base, dev_addr);

The return value from the above function is 0 or -1. 

> +		/* if negative, something failed */
> +		if (ret < 0)
> +			return -1;
> +
> +		if (ret == 0) {

Will it be ok to skip this check?

> +			/* vf_token required to open device file descriptor */
> +			rte_uuid_unparse(uuid_token,
> +					 vf_token, sizeof(vf_token));
> +			snprintf(dev, sizeof(dev),
> +				 "%s vf_token=%s", dev_addr, vf_token);
> +		}

snipped

general comment `can we use EAL LOG for info, warn, error`?

Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

[Vamsi Krishna Attunuru]

Hi Vipin,

Please see inline.

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Varghese, Vipin
> Sent: Wednesday, April 1, 2020 8:45 AM
> To: Vamsi Krishna Attunuru <vattunuru@marvell.com>; dev@dpdk.org
> Cc: Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> alex.williamson@redhat.com; thomas@monjalon.net;
> david.marchand@redhat.com
> Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device
> access
> 
> Snipped
> > diff --git a/lib/librte_eal/linux/eal/eal_vfio.c
> > b/lib/librte_eal/linux/eal/eal_vfio.c
> > index 01b5ef3..e2fdd35 100644
> > --- a/lib/librte_eal/linux/eal/eal_vfio.c
> > +++ b/lib/librte_eal/linux/eal/eal_vfio.c
> 
> DPDK supports freebsd where `/dev/pci` can be probed like Linux
> `/sys/bus/pci`. Will you be adding the functions in
> `lib/librte_eal/freebsd/eal/` folder or implement as `NOT supported`?
> 
> What about window ` lib/librte_eal/windows/eal/`?

If other kernel (freebsd & windows) drivers also support this vf token scheme, functions can be added in respective eal implementations.

> 
> snipped
> > +	snprintf(linkname, sizeof(linkname),
> > +			 "%s/%s/physfn", sysfs_base, dev_addr);
> 
> Would you like to check the pointer sysfs_base or dev_addr?

Physfn pointer is being checked here.
> 
> >  int
> >  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> >  		int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -
> 
> Snipped
> 
> > +		ret = is_vf_token_required(sysfs_base, dev_addr);
> 
> The return value from the above function is 0 or -1.

Function also returns 1.
> 
> > +		/* if negative, something failed */
> > +		if (ret < 0)
> > +			return -1;
> > +
> > +		if (ret == 0) {
> 
> Will it be ok to skip this check?
No, it's a valid case.
> 
> > +			/* vf_token required to open device file descriptor */
> > +			rte_uuid_unparse(uuid_token,
> > +					 vf_token, sizeof(vf_token));
> > +			snprintf(dev, sizeof(dev),
> > +				 "%s vf_token=%s", dev_addr, vf_token);
> > +		}
> 
> snipped
> 
> general comment `can we use EAL LOG for info, warn, error`?

Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

[Wang, Haiyue]

Hi A Vamsi,

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of vattunuru@marvell.com
> Sent: Thursday, March 5, 2020 12:33
> To: dev@dpdk.org
> Cc: jerinj@marvell.com; alex.williamson@redhat.com; thomas@monjalon.net; david.marchand@redhat.com;
> Vamsi Attunuru <vattunuru@marvell.com>
> Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access
> 
> From: Vamsi Attunuru <vattunuru@marvell.com>
> 
> vfio-pci driver enables virtual function access from
> the DPDK applications when those vf device's physical
> function is also bound to vfio driver.
> 
> Patch adds the required configuration and checks to
> enable DPDK applications to access both pf and it's
> vf devices through vfio-pci driver.
> 
> See background on vf token scheme in linux vfio driver.
> http://patches.dpdk.org/cover/65915/
> 
> When a physical function is enabled with  non-zero
> virtual functions,  patch sets the UUID using
> VFIO_DEVICE_FEATURE ioctl from physical function's
> file descriptor. Same UUID is used to gain the access
> for the virtual functions on those  physical function.
> 
> Following changes required on top of this DPDK patch
> * Kernel version check for VFIO_DEVICE_FEATURE ioctl
> * Use uuid gen API to generate UUID.
> 
> Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
> 
> diff --git a/lib/librte_eal/linux/eal/eal_vfio.c b/lib/librte_eal/linux/eal/eal_vfio.c
> index 01b5ef3..e2fdd35 100644
> --- a/lib/librte_eal/linux/eal/eal_vfio.c
> +++ b/lib/librte_eal/linux/eal/eal_vfio.c
> @@ -12,6 +12,7 @@
>  #include <rte_log.h>
>  #include <rte_memory.h>
>  #include <rte_eal_memconfig.h>
> +#include <rte_uuid.h>
>  #include <rte_vfio.h>
> 
>  #include "eal_filesystem.h"
> @@ -50,6 +51,9 @@ struct vfio_config {
>  	struct user_mem_maps mem_maps;
>  };
> 
> +rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
> +				      0x913f, 0xf2d2f965ed0eULL);
> +
>  /* per-process VFIO config */
>  static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];
>  static struct vfio_config *default_vfio_cfg = &vfio_cfgs[0];
> @@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)
>  	return 0;
>  }
> 
> +static bool
> +rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
> +{
> +	char linkname[PATH_MAX];
> +	char filename[PATH_MAX];
> +	int ret;
> +
> +	memset(linkname, 0, sizeof(linkname));
> +	memset(filename, 0, sizeof(filename));
> +
> +	/* check if physfn directory exist for this device */
> +	snprintf(linkname, sizeof(linkname),
> +			 "%s/%s/physfn", sysfs_base, dev_addr);
> +
> +	ret = readlink(linkname, filename, sizeof(filename));
> +
> +	/* For PFs, physfn directory does not exist */
> +	if (ret < 0)
> +		return true;
> +
> +	return false;
> +}
> +
> +static int
> +is_vf_token_required(const char *sysfs_base, const char *dev_addr)
> +{
> +	char *tok[16], *physfn, *physfn_drv;
> +	char linkname[PATH_MAX];
> +	char filename[PATH_MAX];
> +	int ret;
> +
> +	memset(linkname, 0, sizeof(linkname));
> +	memset(filename, 0, sizeof(filename));
> +
> +	snprintf(linkname, sizeof(linkname),
> +			 "%s/%s/physfn", sysfs_base, dev_addr);
> +
> +	ret = readlink(linkname, filename, sizeof(filename));
> +	if (ret < 0)
> +		return -1;
> +
> +	ret = rte_strsplit(filename, sizeof(filename),
> +			tok, RTE_DIM(tok), '/');
> +	if (ret <= 0) {
> +		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
> +		return -1;
> +	}
> +
> +	physfn = tok[ret - 1];
> +
> +	snprintf(linkname, sizeof(linkname),
> +			 "/sys/bus/pci/devices/%s/driver", physfn);
> +	ret = readlink(linkname, filename, sizeof(filename));
> +	if (ret < 0)
> +		return -1;
> +
> +	ret = rte_strsplit(filename, sizeof(filename),
> +			tok, RTE_DIM(tok), '/');
> +	if (ret <= 0) {
> +		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
> +			dev_addr);
> +		return -1;
> +	}
> +
> +	physfn_drv = tok[ret - 1];
> +
> +	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
> +		return 1;
> +
> +	/* physfn is bound to vfio-pci */
> +	return 0;
> +}
> +

Based on Alex's vfio for qemu patch (https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/),
and understand the SR-IOV design by reading your RFC, I scratched a simple design.

1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i

  If no token in VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file-prefix=vf1 -- -i
Kernel error:	vfio-pci 0000:87:02.0: VF token required to access device

./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -- -i


static int
vfio_pci_vf_token_arg_handler(__rte_unused const char *key,
			      const char *value, void *opaque)
{
	if (rte_uuid_parse(value, opaque)) {
		RTE_LOG(ERR, EAL,
			"The VF token is not a valid uuid : %s\n", value);
		return -1;
	}

	return 0;
}

static int
vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
{
	const char *key = "vf_token";
	struct rte_kvargs *kvlist;
	int ret = 0;

	if (devargs == NULL)
		return 0;

	kvlist = rte_kvargs_parse(devargs->args, NULL);
	if (kvlist == NULL)
		return 0;

	if (!rte_kvargs_count(kvlist, key))
		goto exit;

	if (rte_kvargs_process(kvlist, key,
			       vfio_pci_vf_token_arg_handler, uu) < 0)
		goto exit;

	ret = 1;

exit:
	rte_kvargs_free(kvlist);
	return ret;
}



static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +712,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;


int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);



> +static bool
> +rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char *dev_addr)
> +{
> +	char linkname[PATH_MAX];
> +	unsigned long num_vfs;
> +	int ret;
> +
> +	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
> +		return false;
> +
> +	memset(linkname, 0, sizeof(linkname));
> +
> +	snprintf(linkname, sizeof(linkname),
> +			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
> +
> +	ret  = eal_parse_sysfs_value(linkname, &num_vfs);
> +
> +	if ((ret < 0) || (num_vfs == 0))
> +		return false;
> +
> +	return true;
> +}
> +
>  int
>  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  		int *vfio_dev_fd, struct vfio_device_info *device_info)
> @@ -669,6 +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  	int vfio_container_fd;
>  	int vfio_group_fd;
>  	int iommu_group_num;
> +	char dev[PATH_MAX];
>  	int i, ret;
> 
>  	/* get group number */
> @@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  	if (ret < 0)
>  		return -1;
> 
> +	snprintf(dev, sizeof(dev), "%s", dev_addr);
> +
> +	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
> +		char vf_token[PATH_MAX];
> +		/*
> +		 *  Check if vf_token is required or not,
> +		 *  vf_token is required when the VF's physfn is
> +		 *  binded with vfio-pci driver
> +		 */
> +		ret = is_vf_token_required(sysfs_base, dev_addr);
> +		/* if negative, something failed */
> +		if (ret < 0)
> +			return -1;
> +
> +		if (ret == 0) {
> +			/* vf_token required to open device file descriptor */
> +			rte_uuid_unparse(uuid_token,
> +					 vf_token, sizeof(vf_token));
> +			snprintf(dev, sizeof(dev),
> +				 "%s vf_token=%s", dev_addr, vf_token);
> +		}
> +	}
> +
>  	/* get the actual group fd */
>  	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
>  	if (vfio_group_fd < 0)
> @@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  	}
> 
>  	/* get a file descriptor for the device */
> -	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
> +	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
>  	if (*vfio_dev_fd < 0) {
>  		/* if we cannot get a device fd, this implies a problem with
>  		 * the VFIO group or the container not having IOMMU configured.
> @@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  		rte_vfio_clear_group(vfio_group_fd);
>  		return -1;
>  	}
> +
> +	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {
> +#define VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
> +
> +		struct vfio_device_feature *vf_token;
> +		uint8_t local[VF_TOKEN];
> +
> +		memset(local, 0, VF_TOKEN);
> +		vf_token = (struct vfio_device_feature *)local;
> +		vf_token->argsz = VF_TOKEN;
> +		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
> +					VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
> +
> +		memcpy(local + sizeof(struct vfio_device_feature),
> +		       &uuid_token, sizeof(uuid_token));
> +
> +		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
> +		if (ret) {
> +			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
> +					"error %i (%s)\n", dev_addr, errno,
> +					strerror(errno));
> +			return -1;
> +		}
> +	}
> +
>  	vfio_group_device_get(vfio_group_fd);
> 
>  	return 0;
> --
> 2.8.4

Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

[Vamsi Krishna Attunuru]

Hi Wang,

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Wang, Haiyue
> Sent: Thursday, April 2, 2020 6:13 PM
> To: Vamsi Krishna Attunuru <vattunuru@marvell.com>; dev@dpdk.org
> Cc: Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> alex.williamson@redhat.com; thomas@monjalon.net;
> david.marchand@redhat.com
> Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device
> access
> 
> Hi A Vamsi,
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of vattunuru@marvell.com
> > Sent: Thursday, March 5, 2020 12:33
> > To: dev@dpdk.org
> > Cc: jerinj@marvell.com; alex.williamson@redhat.com;
> > thomas@monjalon.net; david.marchand@redhat.com; Vamsi Attunuru
> > <vattunuru@marvell.com>
> > Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device
> > access
> >
> > From: Vamsi Attunuru <vattunuru@marvell.com>
> >
> > vfio-pci driver enables virtual function access from the DPDK
> > applications when those vf device's physical function is also bound to
> > vfio driver.
> >
> > Patch adds the required configuration and checks to enable DPDK
> > applications to access both pf and it's vf devices through vfio-pci
> > driver.
> >
> > See background on vf token scheme in linux vfio driver.
> > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk.org_c
> >
> over_65915_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP02g
> VZIWTVrW
> > -6zNZz5-
> uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk&
> s=
> > tvJ4Li8zpLMbp_p3vKQL7QV09DttNcl70YAr6BVHZm8&e=
> >
> > When a physical function is enabled with  non-zero virtual functions,
> > patch sets the UUID using VFIO_DEVICE_FEATURE ioctl from physical
> > function's file descriptor. Same UUID is used to gain the access for
> > the virtual functions on those  physical function.
> >
> > Following changes required on top of this DPDK patch
> > * Kernel version check for VFIO_DEVICE_FEATURE ioctl
> > * Use uuid gen API to generate UUID.
> >
> > Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
> >
> > diff --git a/lib/librte_eal/linux/eal/eal_vfio.c
> > b/lib/librte_eal/linux/eal/eal_vfio.c
> > index 01b5ef3..e2fdd35 100644
> > --- a/lib/librte_eal/linux/eal/eal_vfio.c
> > +++ b/lib/librte_eal/linux/eal/eal_vfio.c
> > @@ -12,6 +12,7 @@
> >  #include <rte_log.h>
> >  #include <rte_memory.h>
> >  #include <rte_eal_memconfig.h>
> > +#include <rte_uuid.h>
> >  #include <rte_vfio.h>
> >
> >  #include "eal_filesystem.h"
> > @@ -50,6 +51,9 @@ struct vfio_config {
> >  	struct user_mem_maps mem_maps;
> >  };
> >
> > +rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
> > +				      0x913f, 0xf2d2f965ed0eULL);
> > +
> >  /* per-process VFIO config */
> >  static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];  static
> > struct vfio_config *default_vfio_cfg = &vfio_cfgs[0]; @@ -657,6
> > +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)
> >  	return 0;
> >  }
> >
> > +static bool
> > +rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
> > +{
> > +	char linkname[PATH_MAX];
> > +	char filename[PATH_MAX];
> > +	int ret;
> > +
> > +	memset(linkname, 0, sizeof(linkname));
> > +	memset(filename, 0, sizeof(filename));
> > +
> > +	/* check if physfn directory exist for this device */
> > +	snprintf(linkname, sizeof(linkname),
> > +			 "%s/%s/physfn", sysfs_base, dev_addr);
> > +
> > +	ret = readlink(linkname, filename, sizeof(filename));
> > +
> > +	/* For PFs, physfn directory does not exist */
> > +	if (ret < 0)
> > +		return true;
> > +
> > +	return false;
> > +}
> > +
> > +static int
> > +is_vf_token_required(const char *sysfs_base, const char *dev_addr) {
> > +	char *tok[16], *physfn, *physfn_drv;
> > +	char linkname[PATH_MAX];
> > +	char filename[PATH_MAX];
> > +	int ret;
> > +
> > +	memset(linkname, 0, sizeof(linkname));
> > +	memset(filename, 0, sizeof(filename));
> > +
> > +	snprintf(linkname, sizeof(linkname),
> > +			 "%s/%s/physfn", sysfs_base, dev_addr);
> > +
> > +	ret = readlink(linkname, filename, sizeof(filename));
> > +	if (ret < 0)
> > +		return -1;
> > +
> > +	ret = rte_strsplit(filename, sizeof(filename),
> > +			tok, RTE_DIM(tok), '/');
> > +	if (ret <= 0) {
> > +		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
> > +		return -1;
> > +	}
> > +
> > +	physfn = tok[ret - 1];
> > +
> > +	snprintf(linkname, sizeof(linkname),
> > +			 "/sys/bus/pci/devices/%s/driver", physfn);
> > +	ret = readlink(linkname, filename, sizeof(filename));
> > +	if (ret < 0)
> > +		return -1;
> > +
> > +	ret = rte_strsplit(filename, sizeof(filename),
> > +			tok, RTE_DIM(tok), '/');
> > +	if (ret <= 0) {
> > +		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
> > +			dev_addr);
> > +		return -1;
> > +	}
> > +
> > +	physfn_drv = tok[ret - 1];
> > +
> > +	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
> > +		return 1;
> > +
> > +	/* physfn is bound to vfio-pci */
> > +	return 0;
> > +}
> > +
> 
> Based on Alex's vfio for qemu patch
> (https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__lore.kernel.org_lkml_20200204161737.34696b91-
> 40w520.home_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP0
> 2gVZIWTVrW-6zNZz5-
> uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk&
> s=0pgRda84NrL2omZa0mn-ufSEzKksSi440O2AYImZLaI&e= ), and understand
> the SR-IOV design by reading your RFC, I scratched a simple design.

Thanks, looks good to me, I guess now it serves the QEMU based use cases as well.
> 
> 1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0 2. echo 2 >
> /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
> 3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w
> 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf --
> -i
> 
>   If no token in VF:
> ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file-
> prefix=vf1 -- -i
> Kernel error:	vfio-pci 0000:87:02.0: VF token required to access device
> 
> ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w
> 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -
> - -i
> 
> 
> static int
> vfio_pci_vf_token_arg_handler(__rte_unused const char *key,
> 			      const char *value, void *opaque) {
> 	if (rte_uuid_parse(value, opaque)) {
> 		RTE_LOG(ERR, EAL,
> 			"The VF token is not a valid uuid : %s\n", value);
> 		return -1;
> 	}
> 
> 	return 0;
> }
> 
> static int
> vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) {
> 	const char *key = "vf_token";
> 	struct rte_kvargs *kvlist;
> 	int ret = 0;
> 
> 	if (devargs == NULL)
> 		return 0;
> 
> 	kvlist = rte_kvargs_parse(devargs->args, NULL);
> 	if (kvlist == NULL)
> 		return 0;
> 
> 	if (!rte_kvargs_count(kvlist, key))
> 		goto exit;
> 
> 	if (rte_kvargs_process(kvlist, key,
> 			       vfio_pci_vf_token_arg_handler, uu) < 0)
> 		goto exit;
> 
> 	ret = 1;
> 
> exit:
> 	rte_kvargs_free(kvlist);
> 	return ret;
> }
> 
> 
> 
> static int
>  pci_vfio_map_resource_primary(struct rte_pci_device *dev)  {
>  	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> +	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
>  	char pci_addr[PATH_MAX] = {0};
>  	int vfio_dev_fd;
>  	struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +712,9 @@
> pci_vfio_map_resource_primary(struct rte_pci_device *dev)
>  	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
>  			loc->domain, loc->bus, loc->devid, loc->function);
> 
> +	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
>  	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> -					&vfio_dev_fd, &device_info);
> +					&vfio_dev_fd, &device_info,
> vf_token);
>  	if (ret)
>  		return ret;
> 
> 
> int
>  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> -		int *vfio_dev_fd, struct vfio_device_info *device_info)
> +		int *vfio_dev_fd, struct vfio_device_info *device_info,
> +		rte_uuid_t vf_token)
>  {
>  	struct vfio_group_status group_status = {
>  			.argsz = sizeof(group_status)
> @@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const
> char *dev_addr,
>  	int vfio_container_fd;
>  	int vfio_group_fd;
>  	int iommu_group_num;
> +	char dev[PATH_MAX];
>  	int i, ret;
> 
>  	/* get group number */
> @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,
> const char *dev_addr,
>  				t->type_id, t->name);
>  	}
> 
> +	if (!rte_uuid_is_null(vf_token)) {
> +		char vf_token_str[PATH_MAX];
> +
> +		rte_uuid_unparse(vf_token, vf_token_str,
> sizeof(vf_token_str));
> +		snprintf(dev, sizeof(dev),
> +			 "%s vf_token=%s", dev_addr, vf_token_str);
> +	} else {
> +		snprintf(dev, sizeof(dev),
> +			 "%s", dev_addr);
> +	}
> +
>  	/* get a file descriptor for the device */
> -	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> dev_addr);
> +	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> dev);
> 
> 
> 
> > +static bool
> > +rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char
> > +*dev_addr) {
> > +	char linkname[PATH_MAX];
> > +	unsigned long num_vfs;
> > +	int ret;
> > +
> > +	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
> > +		return false;
> > +
> > +	memset(linkname, 0, sizeof(linkname));
> > +
> > +	snprintf(linkname, sizeof(linkname),
> > +			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
> > +
> > +	ret  = eal_parse_sysfs_value(linkname, &num_vfs);
> > +
> > +	if ((ret < 0) || (num_vfs == 0))
> > +		return false;
> > +
> > +	return true;
> > +}
> > +
> >  int
> >  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> >  		int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -
> 669,6
> > +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char
> *dev_addr,
> >  	int vfio_container_fd;
> >  	int vfio_group_fd;
> >  	int iommu_group_num;
> > +	char dev[PATH_MAX];
> >  	int i, ret;
> >
> >  	/* get group number */
> > @@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base,
> const char *dev_addr,
> >  	if (ret < 0)
> >  		return -1;
> >
> > +	snprintf(dev, sizeof(dev), "%s", dev_addr);
> > +
> > +	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
> > +		char vf_token[PATH_MAX];
> > +		/*
> > +		 *  Check if vf_token is required or not,
> > +		 *  vf_token is required when the VF's physfn is
> > +		 *  binded with vfio-pci driver
> > +		 */
> > +		ret = is_vf_token_required(sysfs_base, dev_addr);
> > +		/* if negative, something failed */
> > +		if (ret < 0)
> > +			return -1;
> > +
> > +		if (ret == 0) {
> > +			/* vf_token required to open device file descriptor */
> > +			rte_uuid_unparse(uuid_token,
> > +					 vf_token, sizeof(vf_token));
> > +			snprintf(dev, sizeof(dev),
> > +				 "%s vf_token=%s", dev_addr, vf_token);
> > +		}
> > +	}
> > +
> >  	/* get the actual group fd */
> >  	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
> >  	if (vfio_group_fd < 0)
> > @@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base,
> const char *dev_addr,
> >  	}
> >
> >  	/* get a file descriptor for the device */
> > -	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> dev_addr);
> > +	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> dev);
> >  	if (*vfio_dev_fd < 0) {
> >  		/* if we cannot get a device fd, this implies a problem with
> >  		 * the VFIO group or the container not having IOMMU
> configured.
> > @@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base,
> const char *dev_addr,
> >  		rte_vfio_clear_group(vfio_group_fd);
> >  		return -1;
> >  	}
> > +
> > +	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {
> #define
> > +VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
> > +
> > +		struct vfio_device_feature *vf_token;
> > +		uint8_t local[VF_TOKEN];
> > +
> > +		memset(local, 0, VF_TOKEN);
> > +		vf_token = (struct vfio_device_feature *)local;
> > +		vf_token->argsz = VF_TOKEN;
> > +		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
> > +
> 	VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
> > +
> > +		memcpy(local + sizeof(struct vfio_device_feature),
> > +		       &uuid_token, sizeof(uuid_token));
> > +
> > +		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
> > +		if (ret) {
> > +			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
> > +					"error %i (%s)\n", dev_addr, errno,
> > +					strerror(errno));
> > +			return -1;
> > +		}
> > +	}
> > +
> >  	vfio_group_device_get(vfio_group_fd);
> >
> >  	return 0;
> > --
> > 2.8.4

Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

[Wang, Haiyue]

Hi Vamsi,

Fixed one dev_args issue, and send the whole patch to express the whole design:
https://patchwork.dpdk.org/patch/68114/

BR,
Haiyue

> -----Original Message-----
> From: Vamsi Krishna Attunuru <vattunuru@marvell.com>
> Sent: Friday, April 10, 2020 15:28
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org
> Cc: Jerin Jacob Kollanukkaran <jerinj@marvell.com>; alex.williamson@redhat.com; thomas@monjalon.net;
> david.marchand@redhat.com
> Subject: RE: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access
> 
> Hi Wang,
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Wang, Haiyue
> > Sent: Thursday, April 2, 2020 6:13 PM
> > To: Vamsi Krishna Attunuru <vattunuru@marvell.com>; dev@dpdk.org
> > Cc: Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> > alex.williamson@redhat.com; thomas@monjalon.net;
> > david.marchand@redhat.com
> > Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device
> > access
> >
> > Hi A Vamsi,
> >
> > > -----Original Message-----
> > > From: dev <dev-bounces@dpdk.org> On Behalf Of vattunuru@marvell.com
> > > Sent: Thursday, March 5, 2020 12:33
> > > To: dev@dpdk.org
> > > Cc: jerinj@marvell.com; alex.williamson@redhat.com;
> > > thomas@monjalon.net; david.marchand@redhat.com; Vamsi Attunuru
> > > <vattunuru@marvell.com>
> > > Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device
> > > access
> > >
> > > From: Vamsi Attunuru <vattunuru@marvell.com>
> > >
> > > vfio-pci driver enables virtual function access from the DPDK
> > > applications when those vf device's physical function is also bound to
> > > vfio driver.
> > >
> > > Patch adds the required configuration and checks to enable DPDK
> > > applications to access both pf and it's vf devices through vfio-pci
> > > driver.
> > >
> > > See background on vf token scheme in linux vfio driver.
> > > https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk.org_c
> > >
> > over_65915_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP02g
> > VZIWTVrW
> > > -6zNZz5-
> > uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk&
> > s=
> > > tvJ4Li8zpLMbp_p3vKQL7QV09DttNcl70YAr6BVHZm8&e=
> > >
> > > When a physical function is enabled with  non-zero virtual functions,
> > > patch sets the UUID using VFIO_DEVICE_FEATURE ioctl from physical
> > > function's file descriptor. Same UUID is used to gain the access for
> > > the virtual functions on those  physical function.
> > >
> > > Following changes required on top of this DPDK patch
> > > * Kernel version check for VFIO_DEVICE_FEATURE ioctl
> > > * Use uuid gen API to generate UUID.
> > >
> > > Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>
> > >
> > > diff --git a/lib/librte_eal/linux/eal/eal_vfio.c
> > > b/lib/librte_eal/linux/eal/eal_vfio.c
> > > index 01b5ef3..e2fdd35 100644
> > > --- a/lib/librte_eal/linux/eal/eal_vfio.c
> > > +++ b/lib/librte_eal/linux/eal/eal_vfio.c
> > > @@ -12,6 +12,7 @@
> > >  #include <rte_log.h>
> > >  #include <rte_memory.h>
> > >  #include <rte_eal_memconfig.h>
> > > +#include <rte_uuid.h>
> > >  #include <rte_vfio.h>
> > >
> > >  #include "eal_filesystem.h"
> > > @@ -50,6 +51,9 @@ struct vfio_config {
> > >  	struct user_mem_maps mem_maps;
> > >  };
> > >
> > > +rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
> > > +				      0x913f, 0xf2d2f965ed0eULL);
> > > +
> > >  /* per-process VFIO config */
> > >  static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];  static
> > > struct vfio_config *default_vfio_cfg = &vfio_cfgs[0]; @@ -657,6
> > > +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)
> > >  	return 0;
> > >  }
> > >
> > > +static bool
> > > +rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
> > > +{
> > > +	char linkname[PATH_MAX];
> > > +	char filename[PATH_MAX];
> > > +	int ret;
> > > +
> > > +	memset(linkname, 0, sizeof(linkname));
> > > +	memset(filename, 0, sizeof(filename));
> > > +
> > > +	/* check if physfn directory exist for this device */
> > > +	snprintf(linkname, sizeof(linkname),
> > > +			 "%s/%s/physfn", sysfs_base, dev_addr);
> > > +
> > > +	ret = readlink(linkname, filename, sizeof(filename));
> > > +
> > > +	/* For PFs, physfn directory does not exist */
> > > +	if (ret < 0)
> > > +		return true;
> > > +
> > > +	return false;
> > > +}
> > > +
> > > +static int
> > > +is_vf_token_required(const char *sysfs_base, const char *dev_addr) {
> > > +	char *tok[16], *physfn, *physfn_drv;
> > > +	char linkname[PATH_MAX];
> > > +	char filename[PATH_MAX];
> > > +	int ret;
> > > +
> > > +	memset(linkname, 0, sizeof(linkname));
> > > +	memset(filename, 0, sizeof(filename));
> > > +
> > > +	snprintf(linkname, sizeof(linkname),
> > > +			 "%s/%s/physfn", sysfs_base, dev_addr);
> > > +
> > > +	ret = readlink(linkname, filename, sizeof(filename));
> > > +	if (ret < 0)
> > > +		return -1;
> > > +
> > > +	ret = rte_strsplit(filename, sizeof(filename),
> > > +			tok, RTE_DIM(tok), '/');
> > > +	if (ret <= 0) {
> > > +		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
> > > +		return -1;
> > > +	}
> > > +
> > > +	physfn = tok[ret - 1];
> > > +
> > > +	snprintf(linkname, sizeof(linkname),
> > > +			 "/sys/bus/pci/devices/%s/driver", physfn);
> > > +	ret = readlink(linkname, filename, sizeof(filename));
> > > +	if (ret < 0)
> > > +		return -1;
> > > +
> > > +	ret = rte_strsplit(filename, sizeof(filename),
> > > +			tok, RTE_DIM(tok), '/');
> > > +	if (ret <= 0) {
> > > +		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
> > > +			dev_addr);
> > > +		return -1;
> > > +	}
> > > +
> > > +	physfn_drv = tok[ret - 1];
> > > +
> > > +	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
> > > +		return 1;
> > > +
> > > +	/* physfn is bound to vfio-pci */
> > > +	return 0;
> > > +}
> > > +
> >
> > Based on Alex's vfio for qemu patch
> > (https://urldefense.proofpoint.com/v2/url?u=https-
> > 3A__lore.kernel.org_lkml_20200204161737.34696b91-
> > 40w520.home_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP0
> > 2gVZIWTVrW-6zNZz5-
> > uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk&
> > s=0pgRda84NrL2omZa0mn-ufSEzKksSi440O2AYImZLaI&e= ), and understand
> > the SR-IOV design by reading your RFC, I scratched a simple design.
> 
> Thanks, looks good to me, I guess now it serves the QEMU based use cases as well.
> >
> > 1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0 2. echo 2 >
> > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
> > 3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w
> > 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf --
> > -i
> >
> >   If no token in VF:
> > ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file-
> > prefix=vf1 -- -i
> > Kernel error:	vfio-pci 0000:87:02.0: VF token required to access device
> >
> > ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w
> > 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -
> > - -i
> >
> >
> > static int
> > vfio_pci_vf_token_arg_handler(__rte_unused const char *key,
> > 			      const char *value, void *opaque) {
> > 	if (rte_uuid_parse(value, opaque)) {
> > 		RTE_LOG(ERR, EAL,
> > 			"The VF token is not a valid uuid : %s\n", value);
> > 		return -1;
> > 	}
> >
> > 	return 0;
> > }
> >
> > static int
> > vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) {
> > 	const char *key = "vf_token";
> > 	struct rte_kvargs *kvlist;
> > 	int ret = 0;
> >
> > 	if (devargs == NULL)
> > 		return 0;
> >
> > 	kvlist = rte_kvargs_parse(devargs->args, NULL);
> > 	if (kvlist == NULL)
> > 		return 0;
> >
> > 	if (!rte_kvargs_count(kvlist, key))
> > 		goto exit;
> >
> > 	if (rte_kvargs_process(kvlist, key,
> > 			       vfio_pci_vf_token_arg_handler, uu) < 0)
> > 		goto exit;
> >
> > 	ret = 1;
> >
> > exit:
> > 	rte_kvargs_free(kvlist);
> > 	return ret;
> > }
> >
> >
> >
> > static int
> >  pci_vfio_map_resource_primary(struct rte_pci_device *dev)  {
> >  	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> > +	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
> >  	char pci_addr[PATH_MAX] = {0};
> >  	int vfio_dev_fd;
> >  	struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +712,9 @@
> > pci_vfio_map_resource_primary(struct rte_pci_device *dev)
> >  	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
> >  			loc->domain, loc->bus, loc->devid, loc->function);
> >
> > +	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
> >  	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> > -					&vfio_dev_fd, &device_info);
> > +					&vfio_dev_fd, &device_info,
> > vf_token);
> >  	if (ret)
> >  		return ret;
> >
> >
> > int
> >  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> > -		int *vfio_dev_fd, struct vfio_device_info *device_info)
> > +		int *vfio_dev_fd, struct vfio_device_info *device_info,
> > +		rte_uuid_t vf_token)
> >  {
> >  	struct vfio_group_status group_status = {
> >  			.argsz = sizeof(group_status)
> > @@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const
> > char *dev_addr,
> >  	int vfio_container_fd;
> >  	int vfio_group_fd;
> >  	int iommu_group_num;
> > +	char dev[PATH_MAX];
> >  	int i, ret;
> >
> >  	/* get group number */
> > @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,
> > const char *dev_addr,
> >  				t->type_id, t->name);
> >  	}
> >
> > +	if (!rte_uuid_is_null(vf_token)) {
> > +		char vf_token_str[PATH_MAX];
> > +
> > +		rte_uuid_unparse(vf_token, vf_token_str,
> > sizeof(vf_token_str));
> > +		snprintf(dev, sizeof(dev),
> > +			 "%s vf_token=%s", dev_addr, vf_token_str);
> > +	} else {
> > +		snprintf(dev, sizeof(dev),
> > +			 "%s", dev_addr);
> > +	}
> > +
> >  	/* get a file descriptor for the device */
> > -	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> > dev_addr);
> > +	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> > dev);
> >
> >
> >
> > > +static bool
> > > +rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char
> > > +*dev_addr) {
> > > +	char linkname[PATH_MAX];
> > > +	unsigned long num_vfs;
> > > +	int ret;
> > > +
> > > +	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
> > > +		return false;
> > > +
> > > +	memset(linkname, 0, sizeof(linkname));
> > > +
> > > +	snprintf(linkname, sizeof(linkname),
> > > +			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
> > > +
> > > +	ret  = eal_parse_sysfs_value(linkname, &num_vfs);
> > > +
> > > +	if ((ret < 0) || (num_vfs == 0))
> > > +		return false;
> > > +
> > > +	return true;
> > > +}
> > > +
> > >  int
> > >  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> > >  		int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -
> > 669,6
> > > +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char
> > *dev_addr,
> > >  	int vfio_container_fd;
> > >  	int vfio_group_fd;
> > >  	int iommu_group_num;
> > > +	char dev[PATH_MAX];
> > >  	int i, ret;
> > >
> > >  	/* get group number */
> > > @@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base,
> > const char *dev_addr,
> > >  	if (ret < 0)
> > >  		return -1;
> > >
> > > +	snprintf(dev, sizeof(dev), "%s", dev_addr);
> > > +
> > > +	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
> > > +		char vf_token[PATH_MAX];
> > > +		/*
> > > +		 *  Check if vf_token is required or not,
> > > +		 *  vf_token is required when the VF's physfn is
> > > +		 *  binded with vfio-pci driver
> > > +		 */
> > > +		ret = is_vf_token_required(sysfs_base, dev_addr);
> > > +		/* if negative, something failed */
> > > +		if (ret < 0)
> > > +			return -1;
> > > +
> > > +		if (ret == 0) {
> > > +			/* vf_token required to open device file descriptor */
> > > +			rte_uuid_unparse(uuid_token,
> > > +					 vf_token, sizeof(vf_token));
> > > +			snprintf(dev, sizeof(dev),
> > > +				 "%s vf_token=%s", dev_addr, vf_token);
> > > +		}
> > > +	}
> > > +
> > >  	/* get the actual group fd */
> > >  	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
> > >  	if (vfio_group_fd < 0)
> > > @@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base,
> > const char *dev_addr,
> > >  	}
> > >
> > >  	/* get a file descriptor for the device */
> > > -	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> > dev_addr);
> > > +	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> > dev);
> > >  	if (*vfio_dev_fd < 0) {
> > >  		/* if we cannot get a device fd, this implies a problem with
> > >  		 * the VFIO group or the container not having IOMMU
> > configured.
> > > @@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base,
> > const char *dev_addr,
> > >  		rte_vfio_clear_group(vfio_group_fd);
> > >  		return -1;
> > >  	}
> > > +
> > > +	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {
> > #define
> > > +VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
> > > +
> > > +		struct vfio_device_feature *vf_token;
> > > +		uint8_t local[VF_TOKEN];
> > > +
> > > +		memset(local, 0, VF_TOKEN);
> > > +		vf_token = (struct vfio_device_feature *)local;
> > > +		vf_token->argsz = VF_TOKEN;
> > > +		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
> > > +
> > 	VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
> > > +
> > > +		memcpy(local + sizeof(struct vfio_device_feature),
> > > +		       &uuid_token, sizeof(uuid_token));
> > > +
> > > +		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
> > > +		if (ret) {
> > > +			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
> > > +					"error %i (%s)\n", dev_addr, errno,
> > > +					strerror(errno));
> > > +			return -1;
> > > +		}
> > > +	}
> > > +
> > >  	vfio_group_device_get(vfio_group_fd);
> > >
> > >  	return 0;
> > > --
> > > 2.8.4

[dpdk-dev] [RFC v2] eal: add VFIO-PCI SR-IOV support

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

3. Start the PF:
  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

4. Start the VF:
   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Test based on vfio-pci patch: https://patchwork.ozlabs.org/cover/1253222/

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
Based on RFC v1: https://patchwork.dpdk.org/patch/66281/
---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 4 files changed, 80 insertions(+), 7 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..7f99337c7 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e6..cbb4c400e 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..1f9e22d82 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..8c0ad04e5 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.0

Re: [dpdk-dev] [EXT] [RFC v2] eal: add VFIO-PCI SR-IOV support

[Vamsi Krishna Attunuru]

> -----Original Message-----
> From: Haiyue Wang <haiyue.wang@intel.com>
> Sent: Friday, April 10, 2020 1:03 PM
> To: dev@dpdk.org; thomas@monjalon.net; Vamsi Krishna Attunuru
> <vattunuru@marvell.com>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> alex.williamson@redhat.com; david.marchand@redhat.com
> Cc: Haiyue Wang <haiyue.wang@intel.com>
> Subject: [EXT] [RFC v2] eal: add VFIO-PCI SR-IOV support
> 
> External Email
> 
> ----------------------------------------------------------------------
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support.
> 
> The VF token can be set by a vfio-pci based PF driver and must be known by
> the vfio-pci based VF driver in order to gain access to the device.
> 
> An example VF token option would take this form:
> 
> 1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
> 
> 2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
> 
> 3. Start the PF:
>   ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
>          -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
>          --file-prefix=pf -- -i
> 
> 4. Start the VF:
>    ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
>          -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
>          --file-prefix=vf1 -- -i
> 
> Test based on vfio-pci patch:
> https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.ozlabs.org_cover_1253222_&d=DwIDAg&c=nKjWec2b6R0m
> OyPaz7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SFD0o
> TfFgnU88wT2X7qMWRDen3KFV8oSOlqz3WQm3nI&s=CU2jxKUsy4oiI26apob
> DF8BJebDrheBiu3eKlF3e76E&e=
> 
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> ---
> Based on RFC v1: https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_patch_66281_&d=DwIDAg&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SFD0oTfFg
> nU88wT2X7qMWRDen3KFV8oSOlqz3WQm3nI&s=3dHnLcXpkRVOZs6wdZQylc
> Je0oNNBs77hXE6SOuLO7o&e=
> ---
>  drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
>  lib/librte_eal/freebsd/eal.c      |  3 +-
>  lib/librte_eal/include/rte_vfio.h |  8 ++++-
>  lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
>  4 files changed, 80 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
> index 64cd84a68..7f99337c7 100644
> --- a/drivers/bus/pci/linux/pci_vfio.c
> +++ b/drivers/bus/pci/linux/pci_vfio.c
> @@ -11,6 +11,7 @@
>  #include <sys/mman.h>
>  #include <stdbool.h>
> 
> +#include <rte_devargs.h>
>  #include <rte_log.h>
>  #include <rte_pci.h>
>  #include <rte_bus_pci.h>
> @@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int
> msix_region)
>  	return ret;
>  }
> 
> +static void
> +vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) {
> +#define VF_TOKEN_ARG "vf_token="
> +	char c, *p, *vf_token;
> +
> +	if (devargs == NULL)
> +		return;
> +
> +	p = strstr(devargs->args, VF_TOKEN_ARG);
> +	if (!p)
> +		return;
> +
> +	vf_token = p + strlen(VF_TOKEN_ARG);
> +	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
> +		return;
> +
> +	c = vf_token[RTE_UUID_STRLEN - 1];
> +	if (c != '\0' && c != ',')
> +		return;
> +
> +	vf_token[RTE_UUID_STRLEN - 1] = '\0';
> +	if (rte_uuid_parse(vf_token, uu)) {
> +		RTE_LOG(ERR, EAL,
> +			"The VF token is not a valid uuid : %s\n", vf_token);
> +		vf_token[RTE_UUID_STRLEN - 1] = c;
> +		return;
> +	}
> +
> +	RTE_LOG(DEBUG, EAL,
> +		"The VF token is found : %s\n", vf_token);
> +
> +	vf_token[RTE_UUID_STRLEN - 1] = c;
> +
> +	/* Purge this vfio-pci specific token from the device arguments */
> +	if (c != '\0') {
> +		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
> +		memmove(p, vf_token + RTE_UUID_STRLEN,
> +			strlen(vf_token + RTE_UUID_STRLEN) + 1);
> +	} else {
> +		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
> +		if (p != devargs->args)
> +			p--;
> +
> +		*p = '\0';
> +	}
> +}
> 
>  static int
>  pci_vfio_map_resource_primary(struct rte_pci_device *dev)  {
>  	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> +	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
>  	char pci_addr[PATH_MAX] = {0};
>  	int vfio_dev_fd;
>  	struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +717,9 @@
> pci_vfio_map_resource_primary(struct rte_pci_device *dev)
>  	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
>  			loc->domain, loc->bus, loc->devid, loc->function);
> 
> +	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
>  	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> -					&vfio_dev_fd, &device_info);
> +					&vfio_dev_fd, &device_info,
> vf_token);
>  	if (ret)
>  		return ret;
> 
> @@ -797,6 +847,7 @@ static int
>  pci_vfio_map_resource_secondary(struct rte_pci_device *dev)  {
>  	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> +	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
>  	char pci_addr[PATH_MAX] = {0};
>  	int vfio_dev_fd;
>  	struct rte_pci_addr *loc = &dev->addr; @@ -830,8 +881,9 @@
> pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
>  		return -1;
>  	}
> 
> +	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
>  	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> -					&vfio_dev_fd, &device_info);
> +					&vfio_dev_fd, &device_info,
> vf_token);
>  	if (ret)
>  		return ret;
> 
> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c index
> 6ae37e7e6..cbb4c400e 100644
> --- a/lib/librte_eal/freebsd/eal.c
> +++ b/lib/librte_eal/freebsd/eal.c
> @@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)  int
> rte_vfio_setup_device(__rte_unused const char *sysfs_base,
>  		      __rte_unused const char *dev_addr,
>  		      __rte_unused int *vfio_dev_fd,
> -		      __rte_unused struct vfio_device_info *device_info)
> +		      __rte_unused struct vfio_device_info *device_info
> +		      __rte_unused rte_uuid_t vf_token)
>  {
>  	return -1;
>  }
> diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
> index 20ed8c45a..1f9e22d82 100644
> --- a/lib/librte_eal/include/rte_vfio.h
> +++ b/lib/librte_eal/include/rte_vfio.h
> @@ -16,6 +16,8 @@ extern "C" {
> 
>  #include <stdint.h>
> 
> +#include <rte_uuid.h>
> +
>  /*
>   * determine if VFIO is present on the system
>   */
> @@ -102,13 +104,17 @@ struct vfio_device_info;
>   * @param device_info
>   *   Device information.
>   *
> + * @param vf_token
> + *   VF token.
> + *
>   * @return
>   *   0 on success.
>   *   <0 on failure.
>   *   >1 if the device cannot be managed this way.
>   */
>  int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> -		int *vfio_dev_fd, struct vfio_device_info *device_info);
> +		int *vfio_dev_fd, struct vfio_device_info *device_info,
> +		rte_uuid_t vf_token);
> 
>  /**
>   * Release a device mapped to a VFIO-managed I/O MMU group.
> diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
> index 4502aefed..8c0ad04e5 100644
> --- a/lib/librte_eal/linux/eal_vfio.c
> +++ b/lib/librte_eal/linux/eal_vfio.c
> @@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
> 
>  int
>  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> -		int *vfio_dev_fd, struct vfio_device_info *device_info)
> +		int *vfio_dev_fd, struct vfio_device_info *device_info,
> +		rte_uuid_t vf_token)
>  {
>  	struct vfio_group_status group_status = {
>  			.argsz = sizeof(group_status)
> @@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const
> char *dev_addr,
>  	int vfio_container_fd;
>  	int vfio_group_fd;
>  	int iommu_group_num;
> +	char dev[PATH_MAX];
>  	int i, ret;
> 
>  	/* get group number */
> @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,
> const char *dev_addr,
>  				t->type_id, t->name);
>  	}
> 
> +	if (!rte_uuid_is_null(vf_token)) {
> +		char vf_token_str[RTE_UUID_STRLEN];
> +
> +		rte_uuid_unparse(vf_token, vf_token_str,
> sizeof(vf_token_str));
> +		snprintf(dev, sizeof(dev),
> +			 "%s vf_token=%s", dev_addr, vf_token_str);
> +	} else {
> +		snprintf(dev, sizeof(dev),
> +			 "%s", dev_addr);
> +	}
> +
>  	/* get a file descriptor for the device */
> -	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> dev_addr);
> +	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> dev);
>  	if (*vfio_dev_fd < 0) {
>  		/* if we cannot get a device fd, this implies a problem with
>  		 * the VFIO group or the container not having IOMMU
> configured.
> @@ -2081,7 +2094,8 @@ int
>  rte_vfio_setup_device(__rte_unused const char *sysfs_base,
>  		__rte_unused const char *dev_addr,
>  		__rte_unused int *vfio_dev_fd,
> -		__rte_unused struct vfio_device_info *device_info)
> +		__rte_unused struct vfio_device_info *device_info
> +		__rte_unused rte_uuid_t vf_token)
>  {
>  	return -1;
>  }
> --
> 2.26.0

Acked-by: Vamsi Attunuru <vattunuru@marvell.com>

[dpdk-dev] [PATCH v1] eal: add VFIO-PCI SR-IOV support

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:
  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:
   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
---
v1: Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
	   Based on Vamsi's RFC v1, and Alex's patch for Qemu
	       [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]:
	   Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 4 files changed, 80 insertions(+), 7 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..7f99337c7 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e6..cbb4c400e 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..1f9e22d82 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..8c0ad04e5 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.0

[dpdk-dev] [PATCH v2] eal: add VFIO-PCI SR-IOV support

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:
  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:
   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
---
v2: Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
         Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
	   Based on Vamsi's RFC v1, and Alex's patch for Qemu
	       [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]:
	   Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 4 files changed, 80 insertions(+), 7 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..7f99337c7 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e6..a92584795 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..1f9e22d82 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..8c0ad04e5 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.0

[dpdk-dev] [PATCH v3] eal: add VFIO-PCI SR-IOV support

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:
  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:
   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
---
v3: Fix the Travis build failed:
    (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
    (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: https://patchwork.dpdk.org/patch/68240/ 
         Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
         Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
	   Based on Vamsi's RFC v1, and Alex's patch for Qemu
	       [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]:
	   Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 ++
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 5 files changed, 82 insertions(+), 7 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..7f99337c7 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e6..a92584795 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..1f9e22d82 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..8c0ad04e5 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.0

[dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:
  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:
   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
---
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: https://patchwork.dpdk.org/patch/68254/ 
	Fix the Travis build failed:
       (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
       (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: https://patchwork.dpdk.org/patch/68240/ 
         Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
         Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
	   Based on Vamsi's RFC v1, and Alex's patch for Qemu
	       [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]:
	   Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
---
 devtools/libabigail.abignore      |  3 ++
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 ++
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 6 files changed, 85 insertions(+), 7 deletions(-)

diff --git a/devtools/libabigail.abignore b/devtools/libabigail.abignore
index a59df8f13..d918746b4 100644
--- a/devtools/libabigail.abignore
+++ b/devtools/libabigail.abignore
@@ -11,3 +11,6 @@
         type_kind = enum
         name = rte_crypto_asym_xform_type
         changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
+; Ignore this function which is only relevant to linux for driver
+[suppress_type]
+	name = rte_vfio_setup_device
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..7f99337c7 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e6..a92584795 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..1f9e22d82 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..916082b5d 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.0

Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

[Thomas Monjalon]

Hi,

About the title, I think it does not convey what is new here.
VFIO is not new, SR-IOV is already supported.
The title should mention the new VFIO feature in few simple words.
Is it only about using VFIO for PF?


13/04/2020 10:29, Haiyue Wang:
> v4: 1. Ignore rte_vfio_setup_device ABI check since it is
>        for Linux driver use.
[...]
> +; Ignore this function which is only relevant to linux for driver
> +[suppress_type]
> +	name = rte_vfio_setup_device

Adding such exception for all internal "driver interface" functions
is not scaling. Please use __rte_internal.
I am waiting for the patchset about rte_internal to be reviewed or completed.
As it is not progressing, the decision is to block any patch having
ABI issue because of internal false positive.
Please help, thanks.

Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

[Wang, Haiyue]

Hi Thomas,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Monday, April 13, 2020 20:18
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; vattunuru@marvell.com; jerinj@marvell.com; alex.williamson@redhat.com;
> david.marchand@redhat.com
> Subject: Re: [PATCH v4] eal: add VFIO-PCI SR-IOV support
> 
> Hi,
> 
> About the title, I think it does not convey what is new here.
> VFIO is not new, SR-IOV is already supported.
> The title should mention the new VFIO feature in few simple words.
> Is it only about using VFIO for PF?
> 

For Both. Align with Alex's : vfio-pci: QEMU support for vfio-pci VF tokens

How about: 'eal: support for VFIO-PCI VF tokens' ?

> 
> 13/04/2020 10:29, Haiyue Wang:
> > v4: 1. Ignore rte_vfio_setup_device ABI check since it is
> >        for Linux driver use.
> [...]
> > +; Ignore this function which is only relevant to linux for driver
> > +[suppress_type]
> > +	name = rte_vfio_setup_device
> 
> Adding such exception for all internal "driver interface" functions
> is not scaling. Please use __rte_internal.
> I am waiting for the patchset about rte_internal to be reviewed or completed.
> As it is not progressing, the decision is to block any patch having
> ABI issue because of internal false positive.
> Please help, thanks.
> 

I will drop this patch ABI workaround, and try to catch the '__rte_internal ' design.

Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

[Andrew Rybchenko]

On 4/13/20 11:29 AM, Haiyue Wang wrote:
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support since 5.7.
> 
> The VF token can be set by a vfio-pci based PF driver and must be known
> by the vfio-pci based VF driver in order to gain access to the device.
> 
> An example VF token option would take this form:
> 
> 1. Install vfio-pci with option 'enable_sriov=1'
> 
> 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
> 
> 3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
> 
> 4. Start the PF:
>   ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
>          -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
>          --file-prefix=pf -- -i

Should I get a token from my head? Any?

> 5. Start the VF:
>    ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
>          -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
>          --file-prefix=vf1 -- -i
> 
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
> ---
> v4: 1. Ignore rte_vfio_setup_device ABI check since it is
>        for Linux driver use.
> 
> v3: https://patchwork.dpdk.org/patch/68254/ 
> 	Fix the Travis build failed:
>        (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
>        (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
> 
> v2: https://patchwork.dpdk.org/patch/68240/ 
>          Fix the FreeBSD build error.
> 
> v1: https://patchwork.dpdk.org/patch/68237/
>          Update the commit message.
> 
> RFC v2: https://patchwork.dpdk.org/patch/68114/
> 	   Based on Vamsi's RFC v1, and Alex's patch for Qemu
> 	       [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]:
> 	   Use the devarg to pass-down the VF token.
> 
> RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
> ---
>  devtools/libabigail.abignore      |  3 ++
>  drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
>  lib/librte_eal/freebsd/eal.c      |  3 +-
>  lib/librte_eal/include/rte_uuid.h |  2 ++
>  lib/librte_eal/include/rte_vfio.h |  8 ++++-
>  lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
>  6 files changed, 85 insertions(+), 7 deletions(-)
> 
> diff --git a/devtools/libabigail.abignore b/devtools/libabigail.abignore
> index a59df8f13..d918746b4 100644
> --- a/devtools/libabigail.abignore
> +++ b/devtools/libabigail.abignore
> @@ -11,3 +11,6 @@
>          type_kind = enum
>          name = rte_crypto_asym_xform_type
>          changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
> +; Ignore this function which is only relevant to linux for driver
> +[suppress_type]
> +	name = rte_vfio_setup_device
> diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
> index 64cd84a68..7f99337c7 100644
> --- a/drivers/bus/pci/linux/pci_vfio.c
> +++ b/drivers/bus/pci/linux/pci_vfio.c
> @@ -11,6 +11,7 @@
>  #include <sys/mman.h>
>  #include <stdbool.h>
>  
> +#include <rte_devargs.h>
>  #include <rte_log.h>
>  #include <rte_pci.h>
>  #include <rte_bus_pci.h>
> @@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
>  	return ret;
>  }
>  
> +static void
> +vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
> +{
> +#define VF_TOKEN_ARG "vf_token="
> +	char c, *p, *vf_token;
> +
> +	if (devargs == NULL)
> +		return;
> +
> +	p = strstr(devargs->args, VF_TOKEN_ARG);
> +	if (!p)
> +		return;
> +
> +	vf_token = p + strlen(VF_TOKEN_ARG);
> +	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
> +		return;
> +
> +	c = vf_token[RTE_UUID_STRLEN - 1];
> +	if (c != '\0' && c != ',')
> +		return;
> +
> +	vf_token[RTE_UUID_STRLEN - 1] = '\0';

Is it possible to parse and handle devargs using rte_kvargs.h?

> +	if (rte_uuid_parse(vf_token, uu)) {
> +		RTE_LOG(ERR, EAL,
> +			"The VF token is not a valid uuid : %s\n", vf_token);
> +		vf_token[RTE_UUID_STRLEN - 1] = c;
> +		return;

I think that the function must return error which is handled
by the caller when something bad happens (e.g. invalid
UUID).

> +	}
> +
> +	RTE_LOG(DEBUG, EAL,
> +		"The VF token is found : %s\n", vf_token);
> +
> +	vf_token[RTE_UUID_STRLEN - 1] = c;
> +
> +	/* Purge this vfio-pci specific token from the device arguments */
> +	if (c != '\0') {
> +		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
> +		memmove(p, vf_token + RTE_UUID_STRLEN,
> +			strlen(vf_token + RTE_UUID_STRLEN) + 1);
> +	} else {
> +		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
> +		if (p != devargs->args)
> +			p--;
> +
> +		*p = '\0';
> +	}

Is it really required to purge? Why? If yes, it should be explained in
the comment above.

> +}
>  
>  static int
>  pci_vfio_map_resource_primary(struct rte_pci_device *dev)
>  {
>  	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> +	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);

May be it would be better if vfio_pci_vf_token_arg()
initializes it anyway instead of duplication init
in two places?

>  	char pci_addr[PATH_MAX] = {0};
>  	int vfio_dev_fd;
>  	struct rte_pci_addr *loc = &dev->addr;
> @@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
>  	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
>  			loc->domain, loc->bus, loc->devid, loc->function);
>  
> +	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
>  	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> -					&vfio_dev_fd, &device_info);
> +					&vfio_dev_fd, &device_info, vf_token);
>  	if (ret)
>  		return ret;
>  
> @@ -797,6 +847,7 @@ static int
>  pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
>  {
>  	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> +	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
>  	char pci_addr[PATH_MAX] = {0};
>  	int vfio_dev_fd;
>  	struct rte_pci_addr *loc = &dev->addr;
> @@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
>  		return -1;
>  	}
>  
> +	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
>  	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> -					&vfio_dev_fd, &device_info);
> +					&vfio_dev_fd, &device_info, vf_token);
>  	if (ret)
>  		return ret;
>  
> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> index 6ae37e7e6..a92584795 100644
> --- a/lib/librte_eal/freebsd/eal.c
> +++ b/lib/librte_eal/freebsd/eal.c
> @@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
>  int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
>  		      __rte_unused const char *dev_addr,
>  		      __rte_unused int *vfio_dev_fd,
> -		      __rte_unused struct vfio_device_info *device_info)
> +		      __rte_unused struct vfio_device_info *device_info,
> +		      __rte_unused rte_uuid_t vf_token)
>  {
>  	return -1;
>  }
> diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
> index 044afbdfa..8b42e070a 100644
> --- a/lib/librte_eal/include/rte_uuid.h
> +++ b/lib/librte_eal/include/rte_uuid.h
> @@ -15,6 +15,8 @@ extern "C" {
>  #endif
>  
>  #include <stdbool.h>
> +#include <stddef.h>
> +#include <string.h>
>  
>  /**
>   * Struct describing a Universal Unique Identifier
> diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
> index 20ed8c45a..1f9e22d82 100644
> --- a/lib/librte_eal/include/rte_vfio.h
> +++ b/lib/librte_eal/include/rte_vfio.h
> @@ -16,6 +16,8 @@ extern "C" {
>  
>  #include <stdint.h>
>  
> +#include <rte_uuid.h>
> +
>  /*
>   * determine if VFIO is present on the system
>   */
> @@ -102,13 +104,17 @@ struct vfio_device_info;
>   * @param device_info
>   *   Device information.
>   *
> + * @param vf_token
> + *   VF token.

Such comments are useles and just eat space adding  nothing
useful. Please, make it useful and explain what is behind the
parameter, when it is necessary, why? Should it be specified
for PF case, VF case, both?

> + *
>   * @return
>   *   0 on success.
>   *   <0 on failure.
>   *   >1 if the device cannot be managed this way.
>   */
>  int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> -		int *vfio_dev_fd, struct vfio_device_info *device_info);
> +		int *vfio_dev_fd, struct vfio_device_info *device_info,
> +		rte_uuid_t vf_token);

"rte_uuid_t vf_token" looks confusing. Shouldn't it be
"rte_uuid_t *vf_token"?

>  
>  /**
>   * Release a device mapped to a VFIO-managed I/O MMU group.
> diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
> index 4502aefed..916082b5d 100644
> --- a/lib/librte_eal/linux/eal_vfio.c
> +++ b/lib/librte_eal/linux/eal_vfio.c
> @@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
>  
>  int
>  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> -		int *vfio_dev_fd, struct vfio_device_info *device_info)
> +		int *vfio_dev_fd, struct vfio_device_info *device_info,
> +		rte_uuid_t vf_token)
>  {
>  	struct vfio_group_status group_status = {
>  			.argsz = sizeof(group_status)
> @@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  	int vfio_container_fd;
>  	int vfio_group_fd;
>  	int iommu_group_num;
> +	char dev[PATH_MAX];

Why PATH_MAX?

>  	int i, ret;
>  
>  	/* get group number */
> @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  				t->type_id, t->name);
>  	}
>  
> +	if (!rte_uuid_is_null(vf_token)) {
> +		char vf_token_str[RTE_UUID_STRLEN];
> +
> +		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
> +		snprintf(dev, sizeof(dev),
> +			 "%s vf_token=%s", dev_addr, vf_token_str);
> +	} else {
> +		snprintf(dev, sizeof(dev),
> +			 "%s", dev_addr);
> +	}
> +
>  	/* get a file descriptor for the device */
> -	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
> +	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
>  	if (*vfio_dev_fd < 0) {
>  		/* if we cannot get a device fd, this implies a problem with
>  		 * the VFIO group or the container not having IOMMU configured.
> @@ -2081,7 +2094,8 @@ int
>  rte_vfio_setup_device(__rte_unused const char *sysfs_base,
>  		__rte_unused const char *dev_addr,
>  		__rte_unused int *vfio_dev_fd,
> -		__rte_unused struct vfio_device_info *device_info)
> +		__rte_unused struct vfio_device_info *device_info,
> +		__rte_unused rte_uuid_t vf_token)
>  {
>  	return -1;
>  }
> 

Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

[Wang, Haiyue]

Hi Andrew,

> -----Original Message-----
> From: Andrew Rybchenko <arybchenko@solarflare.com>
> Sent: Monday, April 13, 2020 23:38
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; vattunuru@marvell.com;
> jerinj@marvell.com; alex.williamson@redhat.com; david.marchand@redhat.com
> Subject: Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support
> 
> On 4/13/20 11:29 AM, Haiyue Wang wrote:
> > The kernel module vfio-pci introduces the VF token to enable SR-IOV
> > support since 5.7.
> >
> > The VF token can be set by a vfio-pci based PF driver and must be known
> > by the vfio-pci based VF driver in order to gain access to the device.
> >
> > An example VF token option would take this form:
> >
> > 1. Install vfio-pci with option 'enable_sriov=1'
> >
> > 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
> >
> > 3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
> >
> > 4. Start the PF:
> >   ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
> >          -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
> >          --file-prefix=pf -- -i
> 
> Should I get a token from my head? Any?
> 
> > 5. Start the VF:
> >    ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
> >          -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
> >          --file-prefix=vf1 -- -i
> >
> > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
> > ---
> > v4: 1. Ignore rte_vfio_setup_device ABI check since it is
> >        for Linux driver use.
> >
> > v3: https://patchwork.dpdk.org/patch/68254/
> > 	Fix the Travis build failed:
> >        (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
> >        (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
> >
> > v2: https://patchwork.dpdk.org/patch/68240/
> >          Fix the FreeBSD build error.
> >
> > v1: https://patchwork.dpdk.org/patch/68237/
> >          Update the commit message.
> >
> > RFC v2: https://patchwork.dpdk.org/patch/68114/
> > 	   Based on Vamsi's RFC v1, and Alex's patch for Qemu
> > 	       [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]:
> > 	   Use the devarg to pass-down the VF token.
> >
> > RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
> > ---
> >  devtools/libabigail.abignore      |  3 ++
> >  drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
> >  lib/librte_eal/freebsd/eal.c      |  3 +-
> >  lib/librte_eal/include/rte_uuid.h |  2 ++
> >  lib/librte_eal/include/rte_vfio.h |  8 ++++-
> >  lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
> >  6 files changed, 85 insertions(+), 7 deletions(-)
> >
> > diff --git a/devtools/libabigail.abignore b/devtools/libabigail.abignore
> > index a59df8f13..d918746b4 100644
> > --- a/devtools/libabigail.abignore
> > +++ b/devtools/libabigail.abignore
> > @@ -11,3 +11,6 @@
> >          type_kind = enum
> >          name = rte_crypto_asym_xform_type
> >          changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
> > +; Ignore this function which is only relevant to linux for driver
> > +[suppress_type]
> > +	name = rte_vfio_setup_device
> > diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
> > index 64cd84a68..7f99337c7 100644
> > --- a/drivers/bus/pci/linux/pci_vfio.c
> > +++ b/drivers/bus/pci/linux/pci_vfio.c
> > @@ -11,6 +11,7 @@
> >  #include <sys/mman.h>
> >  #include <stdbool.h>
> >
> > +#include <rte_devargs.h>
> >  #include <rte_log.h>
> >  #include <rte_pci.h>
> >  #include <rte_bus_pci.h>
> > @@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
> >  	return ret;
> >  }
> >
> > +static void
> > +vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
> > +{
> > +#define VF_TOKEN_ARG "vf_token="
> > +	char c, *p, *vf_token;
> > +
> > +	if (devargs == NULL)
> > +		return;
> > +
> > +	p = strstr(devargs->args, VF_TOKEN_ARG);
> > +	if (!p)
> > +		return;
> > +
> > +	vf_token = p + strlen(VF_TOKEN_ARG);
> > +	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
> > +		return;
> > +
> > +	c = vf_token[RTE_UUID_STRLEN - 1];
> > +	if (c != '\0' && c != ',')
> > +		return;
> > +
> > +	vf_token[RTE_UUID_STRLEN - 1] = '\0';
> 
> Is it possible to parse and handle devargs using rte_kvargs.h?
> 

Since it needs to remove the 'vf_token', as 'vf_token' is not a
valid PMD related args, so need to parse and delete it.

rte_kvargs_parse(const char *args, const char * const valid_keys[])

> > +	if (rte_uuid_parse(vf_token, uu)) {
> > +		RTE_LOG(ERR, EAL,
> > +			"The VF token is not a valid uuid : %s\n", vf_token);
> > +		vf_token[RTE_UUID_STRLEN - 1] = c;
> > +		return;
> 
> I think that the function must return error which is handled
> by the caller when something bad happens (e.g. invalid
> UUID).
> 

Yes, make sense, will add the error handling.

> > +	}
> > +
> > +	RTE_LOG(DEBUG, EAL,
> > +		"The VF token is found : %s\n", vf_token);
> > +
> > +	vf_token[RTE_UUID_STRLEN - 1] = c;
> > +
> > +	/* Purge this vfio-pci specific token from the device arguments */
> > +	if (c != '\0') {
> > +		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
> > +		memmove(p, vf_token + RTE_UUID_STRLEN,
> > +			strlen(vf_token + RTE_UUID_STRLEN) + 1);
> > +	} else {
> > +		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
> > +		if (p != devargs->args)
> > +			p--;
> > +
> > +		*p = '\0';
> > +	}
> 
> Is it really required to purge? Why? If yes, it should be explained in
> the comment above.

Please see above reply.

> 
> > +}
> >
> >  static int
> >  pci_vfio_map_resource_primary(struct rte_pci_device *dev)
> >  {
> >  	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> > +	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
> 
> May be it would be better if vfio_pci_vf_token_arg()
> initializes it anyway instead of duplication init
> in two places?

+1, will update it.

> 
> >  	char pci_addr[PATH_MAX] = {0};
> >  	int vfio_dev_fd;
> >  	struct rte_pci_addr *loc = &dev->addr;
> > @@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
> >  	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
> >  			loc->domain, loc->bus, loc->devid, loc->function);
> >
> > +	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
> >  	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> > -					&vfio_dev_fd, &device_info);
> > +					&vfio_dev_fd, &device_info, vf_token);
> >  	if (ret)
> >  		return ret;
> >
> > @@ -797,6 +847,7 @@ static int
> >  pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
> >  {
> >  	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
> > +	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
> >  	char pci_addr[PATH_MAX] = {0};
> >  	int vfio_dev_fd;
> >  	struct rte_pci_addr *loc = &dev->addr;
> > @@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
> >  		return -1;
> >  	}
> >
> > +	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
> >  	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
> > -					&vfio_dev_fd, &device_info);
> > +					&vfio_dev_fd, &device_info, vf_token);
> >  	if (ret)
> >  		return ret;
> >
> > diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> > index 6ae37e7e6..a92584795 100644
> > --- a/lib/librte_eal/freebsd/eal.c
> > +++ b/lib/librte_eal/freebsd/eal.c
> > @@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
> >  int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
> >  		      __rte_unused const char *dev_addr,
> >  		      __rte_unused int *vfio_dev_fd,
> > -		      __rte_unused struct vfio_device_info *device_info)
> > +		      __rte_unused struct vfio_device_info *device_info,
> > +		      __rte_unused rte_uuid_t vf_token)
> >  {
> >  	return -1;
> >  }
> > diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
> > index 044afbdfa..8b42e070a 100644
> > --- a/lib/librte_eal/include/rte_uuid.h
> > +++ b/lib/librte_eal/include/rte_uuid.h
> > @@ -15,6 +15,8 @@ extern "C" {
> >  #endif
> >
> >  #include <stdbool.h>
> > +#include <stddef.h>
> > +#include <string.h>
> >
> >  /**
> >   * Struct describing a Universal Unique Identifier
> > diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
> > index 20ed8c45a..1f9e22d82 100644
> > --- a/lib/librte_eal/include/rte_vfio.h
> > +++ b/lib/librte_eal/include/rte_vfio.h
> > @@ -16,6 +16,8 @@ extern "C" {
> >
> >  #include <stdint.h>
> >
> > +#include <rte_uuid.h>
> > +
> >  /*
> >   * determine if VFIO is present on the system
> >   */
> > @@ -102,13 +104,17 @@ struct vfio_device_info;
> >   * @param device_info
> >   *   Device information.
> >   *
> > + * @param vf_token
> > + *   VF token.
> 
> Such comments are useles and just eat space adding  nothing
> useful. Please, make it useful and explain what is behind the
> parameter, when it is necessary, why? Should it be specified
> for PF case, VF case, both?
> 

Will add more comments, yes for both PF and VF, as Alex's linux patch
explained.

> > + *
> >   * @return
> >   *   0 on success.
> >   *   <0 on failure.
> >   *   >1 if the device cannot be managed this way.
> >   */
> >  int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> > -		int *vfio_dev_fd, struct vfio_device_info *device_info);
> > +		int *vfio_dev_fd, struct vfio_device_info *device_info,
> > +		rte_uuid_t vf_token);
> 
> "rte_uuid_t vf_token" looks confusing. Shouldn't it be
> "rte_uuid_t *vf_token"?

This is UUID API design and type definition:

bool rte_uuid_is_null(const rte_uuid_t uu);

DPDK: typedef unsigned char rte_uuid_t[16];

vs

Linux: typedef struct {
	__u8 b[UUID_SIZE];
} uuid_t;

> 
> >
> >  /**
> >   * Release a device mapped to a VFIO-managed I/O MMU group.
> > diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
> > index 4502aefed..916082b5d 100644
> > --- a/lib/librte_eal/linux/eal_vfio.c
> > +++ b/lib/librte_eal/linux/eal_vfio.c
> > @@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
> >
> >  int
> >  rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> > -		int *vfio_dev_fd, struct vfio_device_info *device_info)
> > +		int *vfio_dev_fd, struct vfio_device_info *device_info,
> > +		rte_uuid_t vf_token)
> >  {
> >  	struct vfio_group_status group_status = {
> >  			.argsz = sizeof(group_status)
> > @@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> >  	int vfio_container_fd;
> >  	int vfio_group_fd;
> >  	int iommu_group_num;
> > +	char dev[PATH_MAX];
> 
> Why PATH_MAX?

Based on Vamsi's RFC v1, and found that it looked a little reasonable, ' char pci_addr[PATH_MAX] = {0}; '

static int
pci_vfio_map_resource_primary(struct rte_pci_device *dev)
{
	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
	char pci_addr[PATH_MAX] = {0}; <----

> 
> >  	int i, ret;
> >
> >  	/* get group number */
> > @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
> >  				t->type_id, t->name);
> >  	}
> >
> > +	if (!rte_uuid_is_null(vf_token)) {
> > +		char vf_token_str[RTE_UUID_STRLEN];
> > +
> > +		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
> > +		snprintf(dev, sizeof(dev),
> > +			 "%s vf_token=%s", dev_addr, vf_token_str);
> > +	} else {
> > +		snprintf(dev, sizeof(dev),
> > +			 "%s", dev_addr);
> > +	}
> > +
> >  	/* get a file descriptor for the device */
> > -	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
> > +	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
> >  	if (*vfio_dev_fd < 0) {
> >  		/* if we cannot get a device fd, this implies a problem with
> >  		 * the VFIO group or the container not having IOMMU configured.
> > @@ -2081,7 +2094,8 @@ int
> >  rte_vfio_setup_device(__rte_unused const char *sysfs_base,
> >  		__rte_unused const char *dev_addr,
> >  		__rte_unused int *vfio_dev_fd,
> > -		__rte_unused struct vfio_device_info *device_info)
> > +		__rte_unused struct vfio_device_info *device_info,
> > +		__rte_unused rte_uuid_t vf_token)
> >  {
> >  	return -1;
> >  }
> >

[dpdk-dev] [PATCH v5 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.
    https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
    https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.
         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.
        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/ 
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 +
 lib/librte_eal/include/rte_vfio.h | 21 ++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 5 files changed, 113 insertions(+), 7 deletions(-)

-- 
2.26.0

[dpdk-dev] [PATCH v5 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.0

[dpdk-dev] [PATCH v5 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:
  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:
   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h | 21 ++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 4 files changed, 111 insertions(+), 7 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..efb64e2ba 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return 0;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1)) {
+		RTE_LOG(ERR, EAL, "The VF token length is too short\n");
+		return -1;
+	}
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',') {
+		RTE_LOG(ERR, EAL,
+			"The VF token ends with a invalid character : %c\n", c);
+		return -1;
+	}
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uuid)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is invalid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return -1;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* This VF token will be treated as a invalid device argument if the
+	 * PMD calls the rte_devargs parse API with its own valid argument list,
+	 * so it needs to purge this vfio-pci specific argument.
+	 */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+
+	return 0;
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e6..a92584795 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..ec9de23dc 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,30 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..916082b5d 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.0

[dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.
    https://patchwork.dpdk.org/cover/68364/

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.
    https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
    https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.
         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.
        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/ 
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 +
 lib/librte_eal/include/rte_vfio.h | 21 ++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 5 files changed, 113 insertions(+), 7 deletions(-)

-- 
2.26.0

[dpdk-dev] [PATCH v6 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.0

[dpdk-dev] [PATCH v6 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:
  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:
   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h | 21 ++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 4 files changed, 111 insertions(+), 7 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..efb64e2ba 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return 0;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1)) {
+		RTE_LOG(ERR, EAL, "The VF token length is too short\n");
+		return -1;
+	}
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',') {
+		RTE_LOG(ERR, EAL,
+			"The VF token ends with a invalid character : %c\n", c);
+		return -1;
+	}
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uuid)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is invalid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return -1;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* This VF token will be treated as a invalid device argument if the
+	 * PMD calls the rte_devargs parse API with its own valid argument list,
+	 * so it needs to purge this vfio-pci specific argument.
+	 */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+
+	return 0;
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 6ae37e7e6..a92584795 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..ec9de23dc 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,30 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..916082b5d 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.0

Re: [dpdk-dev] [EXT] [PATCH v6 0/2] support for VFIO-PCI VF token interface

[Vamsi Krishna Attunuru]

> -----Original Message-----
> From: Haiyue Wang <haiyue.wang@intel.com>
> Sent: Tuesday, April 14, 2020 8:51 AM
> To: dev@dpdk.org; thomas@monjalon.net; Vamsi Krishna Attunuru
> <vattunuru@marvell.com>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>;
> alex.williamson@redhat.com; david.marchand@redhat.com
> Cc: Haiyue Wang <haiyue.wang@intel.com>
> Subject: [EXT] [PATCH v6 0/2] support for VFIO-PCI VF token interface
> 
> External Email
> 
> ----------------------------------------------------------------------
> v6: Drop the Fixes tag in uuid, since the file has been
>     moved to another place, not suitable to apply on stable.
>     And this is not a bug, just some kind of enhancement.
> 
> v5: 1. Add the VF token parse error handling.
>     2. Split into two patches for different logic module.
>     3. Add more comments into the code for explaining the design.
>     4. Drop the ABI change workaround, this patch set focuses on code review.
>     https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_cover_68364_&d=DwIDaQ&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9
> V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=mAkB5KtaFDGFdmqfNZV4dy
> w1A8Xy61tm22TNM8kGSrQ&e=
> 
> v4: 1. Ignore rte_vfio_setup_device ABI check since it is
>        for Linux driver use.
>     https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_patch_68255_&d=DwIDaQ&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9
> V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=Qv5ArXmk_WeQn1bTnM6JS
> XpNp3NEhuenHP1VsjUPzGE&e=
> 
> v3: Fix the Travis build failed:
>            (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
>            (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
>     https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_patch_68254_&d=DwIDaQ&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9
> V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=8U5HwQ-j-
> dkzQPk3E99wJauItTQLH0f48qay37pnTZ8&e=
> 
> v2: Fix the FreeBSD build error.
>          https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_patch_68240_&d=DwIDaQ&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9
> V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=7gq0iGeXay4YQ4ydHRdCZ9_
> oWDRQFtGOxGu3chq0z_s&e=
> 
> v1: Update the commit message.
>         https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_patch_68237_&d=DwIDaQ&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9
> V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=ibSeCUMMleRvQRH9WvJDYV
> TfBAt-Fi4lgfdTBOZ5xwc&e=
> 
> RFC v2: https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_patch_68114_&d=DwIDaQ&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9
> V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=JmSNjqCLE3Xh0_VymptqrViX
> VlRgpFN_PZMvM6C2X2s&e=
>          Based on Vamsi's RFC v1, and Alex's patch for Qemu
>         [https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__lore.kernel.org_lkml_20200204161737.34696b91-
> 40w520.home_&d=DwIDaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=WllrYaumVkx
> aWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9V0iyd_fAAZm4bU3jimkCD
> 0khdQeabCt9pESg&s=7tRxVtn0y5xpxqC38hgAB7Bjz7kHR8mbD-
> pIFgP1E7o&e= ]:
>        Use the devarg to pass-down the VF token.
> 
> RFC v1: https://urldefense.proofpoint.com/v2/url?u=https-
> 3A__patchwork.dpdk.org_patch_66281_&d=DwIDaQ&c=nKjWec2b6R0mOyP
> az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9
> V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=riYbXycUMPQxA30y75nRzs08
> f2wM-uAGcQEnK9ENR7w&e=  by Vamsi.
> 
> Haiyue Wang (2):
>   eal: add uuid dependent header files explicitly
>   eal: support for VFIO-PCI VF token
> 
>  drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
>  lib/librte_eal/freebsd/eal.c      |  3 +-
>  lib/librte_eal/include/rte_uuid.h |  2 +  lib/librte_eal/include/rte_vfio.h | 21
> ++++++++-
>  lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
>  5 files changed, 113 insertions(+), 7 deletions(-)
> 
> --
> 2.26.0

Tested following combinations:
* Both PF & VFs are with DPDK (Single process & multi process)
* PF is with DPDK & VFs are with Kernel driver
* PF is with kernel driver & VFs are with DPDK

Tested-by: Vamsi Attunuru <vattunuru@marvell.com>

Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

[David Marchand]

Thanks a lot for working on this.

I can see the support on the kernel side has been pulled in what will be 5.7.
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=2fb732b33b0f4d73f391066c0a970b942c600dab

I'd like to get the dpdk support in 20.05, but for this, can I have a
summary of the kernel discussions?
Is there anything missing/still wip in the kernel?


On the series itself, I did not look in detail, but there is no
documentation on how to use the feature.
We are missing a release note update.
And a small comment on the last changes:

On Tue, Apr 14, 2020 at 5:26 AM Haiyue Wang <haiyue.wang@intel.com> wrote:
>
> v6: Drop the Fixes tag in uuid, since the file has been
>     moved to another place, not suitable to apply on stable.

This is *not* a reason to drop the Fixes: tag.
Hopefully, we still fix bugs once files have been moved.

>     And this is not a bug, just some kind of enhancement.

But on this point, I suppose this is ok not to backport.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

[Wang, Haiyue]

Hi David,

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Saturday, April 18, 2020 17:22
> To: Wang, Haiyue <haiyue.wang@intel.com>; Vamsi Attunuru <vattunuru@marvell.com>;
> alex.williamson@redhat.com
> Cc: dev <dev@dpdk.org>; Thomas Monjalon <thomas@monjalon.net>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>
> Subject: Re: [PATCH v6 0/2] support for VFIO-PCI VF token interface
> 
> Thanks a lot for working on this.
> 
> I can see the support on the kernel side has been pulled in what will be 5.7.
> https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=2fb732b33b0f4d73f391066c
> 0a970b942c600dab
> 
> I'd like to get the dpdk support in 20.05, but for this, can I have a
> summary of the kernel discussions?
> Is there anything missing/still wip in the kernel?
> 
> 
> On the series itself, I did not look in detail, but there is no
> documentation on how to use the feature.
> We are missing a release note update.

If needed in 20.05, I can add it in release note. And about how to,
I just added it in commit message, and some comments in the API
' rte_vfio_setup_device '.

Not sure which document part is reasonable for adding this, any good
suggestion ?


> And a small comment on the last changes:
> 
> On Tue, Apr 14, 2020 at 5:26 AM Haiyue Wang <haiyue.wang@intel.com> wrote:
> >
> > v6: Drop the Fixes tag in uuid, since the file has been
> >     moved to another place, not suitable to apply on stable.
> 
> This is *not* a reason to drop the Fixes: tag.
> Hopefully, we still fix bugs once files have been moved.
> 

OK, will add the tag later.

> >     And this is not a bug, just some kind of enhancement.
> 
> But on this point, I suppose this is ok not to backport.
> 
> 
> --
> David Marchand

Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

[Thomas Monjalon]

18/04/2020 11:38, Wang, Haiyue:
> From: David Marchand <david.marchand@redhat.com>
> > On the series itself, I did not look in detail, but there is no
> > documentation on how to use the feature.
> > We are missing a release note update.
> 
> If needed in 20.05, I can add it in release note. And about how to,
> I just added it in commit message, and some comments in the API
> ' rte_vfio_setup_device '.
> 
> Not sure which document part is reasonable for adding this, any good
> suggestion ?

Here: http://doc.dpdk.org/guides/linux_gsg/linux_drivers.html#vfio

Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

[Wang, Haiyue]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Saturday, April 18, 2020 17:50
> To: David Marchand <david.marchand@redhat.com>; Vamsi Attunuru <vattunuru@marvell.com>;
> alex.williamson@redhat.com; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev <dev@dpdk.org>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> Subject: Re: [PATCH v6 0/2] support for VFIO-PCI VF token interface
> 
> 18/04/2020 11:38, Wang, Haiyue:
> > From: David Marchand <david.marchand@redhat.com>
> > > On the series itself, I did not look in detail, but there is no
> > > documentation on how to use the feature.
> > > We are missing a release note update.
> >
> > If needed in 20.05, I can add it in release note. And about how to,
> > I just added it in commit message, and some comments in the API
> > ' rte_vfio_setup_device '.
> >
> > Not sure which document part is reasonable for adding this, any good
> > suggestion ?
> 
> Here: http://doc.dpdk.org/guides/linux_gsg/linux_drivers.html#vfio
> 
>

OK.

[dpdk-dev] [PATCH v7 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.
    https://patchwork.dpdk.org/cover/68367/

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.
    https://patchwork.dpdk.org/cover/68364/

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.
    https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
    https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.
         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.
        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/ 
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst | 23 +++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_uuid.h      |  2 +
 lib/librte_eal/include/rte_vfio.h      | 21 +++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 7 files changed, 140 insertions(+), 8 deletions(-)

-- 
2.26.1

[dpdk-dev] [PATCH v7 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.1

[dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
Tested-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 doc/guides/linux_gsg/linux_drivers.rst | 23 +++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_vfio.h      | 21 +++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 6 files changed, 138 insertions(+), 8 deletions(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index 238f3e900..97b83a287 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -72,11 +72,32 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The Linux kernel since version 5.7 supports the creation of virtual functions, and by default this feature is off.
+After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between SR-IOV PF and VFs.
+The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain access to the device.
+DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its related VFs.
+
+.. code-block:: console
+
+    1. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
+            --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
+            --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/rel_notes/release_20_05.rst b/doc/guides/rel_notes/release_20_05.rst
index 184967844..9f4a74693 100644
--- a/doc/guides/rel_notes/release_20_05.rst
+++ b/doc/guides/rel_notes/release_20_05.rst
@@ -81,6 +81,11 @@ New Features
   by making use of the event device capabilities. The event mode currently supports
   only inline IPsec protocol offload.
 
+* **Added the creation of virtual functions support for vfio-pci module.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the device by appending the VF token.
 
 Removed Items
 -------------
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..efb64e2ba 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return 0;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1)) {
+		RTE_LOG(ERR, EAL, "The VF token length is too short\n");
+		return -1;
+	}
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',') {
+		RTE_LOG(ERR, EAL,
+			"The VF token ends with a invalid character : %c\n", c);
+		return -1;
+	}
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uuid)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is invalid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return -1;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* This VF token will be treated as a invalid device argument if the
+	 * PMD calls the rte_devargs parse API with its own valid argument list,
+	 * so it needs to purge this vfio-pci specific argument.
+	 */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+
+	return 0;
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 80dc9aa78..86d5a5f49 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..e5476ec6d 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,30 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..916082b5d 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.1

Re: [dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token

[Thomas Monjalon]

18/04/2020 13:16, Haiyue Wang:
> --- a/doc/guides/linux_gsg/linux_drivers.rst
> +++ b/doc/guides/linux_gsg/linux_drivers.rst
> +The Linux kernel since version 5.7 supports the creation of virtual functions, and by default this feature is off.

Creation of virtual function has always been supported.
You need to be more specific about the new use case please.

> +After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between SR-IOV PF and VFs.
> +The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain access to the device.
> +DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its related VFs.

You need to explain where the token comes from. Can it be any random UUID?

Re: [dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token

[Wang, Haiyue]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Saturday, April 18, 2020 21:27
> To: alex.williamson@redhat.com; david.marchand@redhat.com; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>; vattunuru@marvell.com;
> jerinj@marvell.com
> Subject: Re: [PATCH v7 2/2] eal: support for VFIO-PCI VF token
> 
> 18/04/2020 13:16, Haiyue Wang:
> > --- a/doc/guides/linux_gsg/linux_drivers.rst
> > +++ b/doc/guides/linux_gsg/linux_drivers.rst
> > +The Linux kernel since version 5.7 supports the creation of virtual functions, and by default this
> feature is off.
> 
> Creation of virtual function has always been supported.
> You need to be more specific about the new use case please.
> 
> > +After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between
> SR-IOV PF and VFs.
> > +The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain
> access to the device.
> > +DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its
> related VFs.
> 
> You need to explain where the token comes from. Can it be any random UUID?
> 

Try to add more in v8, please review.

[dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.
    https://patchwork.dpdk.org/cover/68845/

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.
    https://patchwork.dpdk.org/cover/68367/

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.
    https://patchwork.dpdk.org/cover/68364/

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.
    https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
    https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.
         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.
        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/ 
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst | 38 ++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_uuid.h      |  2 +
 lib/librte_eal/include/rte_vfio.h      | 21 +++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 7 files changed, 155 insertions(+), 8 deletions(-)

-- 
2.26.1

[dpdk-dev] [PATCH v8 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.1

[dpdk-dev] [PATCH v8 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
Tested-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 doc/guides/linux_gsg/linux_drivers.rst | 38 ++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_vfio.h      | 21 +++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 6 files changed, 153 insertions(+), 8 deletions(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index 238f3e900..3391723c9 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -72,11 +72,47 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual functions, this feature is disabled
+by default. When enabled, the PF needs a shared VF token (UUID) to setup the trust between SR-IOV PF and VFs. The VF
+token is any kind of valid UUID value selected by the user. When the PF device is bound to ``vfio-pci`` module, it should
+not have any VFs created, this is consistent as before for security reason.
+
+Some use cases about how to use the VF token:
+
+ - The user just uses PF only for DPDK, then no VF token is required to start the PF device.
+
+ - The user wants to creat SR-IOV VFs on the PF device which is bound to ``vfio-pci`` module, then the user needs to select
+   a valid UUID type VF token to start the PF device; after the VFs are created, this VF token is also required to access each
+   VF device.
+
+ - If the DPDK application that runs on PF device exits, and the user wants to start it with another different VF token
+   value, it will be OK if no application (DPDP or KVM) runs on VF, otherwise, it will fail to start with a kernel message
+   "[19145.688094] vfio-pci 0000:87:00.0: Incorrect VF token provided for device" shown. When all of the VFs are free, the
+   user can select a new VF token to start the PF device.
+
+The VFs created are bound to ``vfio-pci`` module automatically. DPDK will use the keyword ``vf_token`` as the device argument
+to pass the VF token value to PF and its related VFs, the PMD should not use it, and this argument will be pruned from the
+device argument list, so the PMD can parse its own valid device arguments successfully without seeing it.
+
+.. code-block:: console
+
+    1. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/rel_notes/release_20_05.rst b/doc/guides/rel_notes/release_20_05.rst
index 184967844..0a30f912b 100644
--- a/doc/guides/rel_notes/release_20_05.rst
+++ b/doc/guides/rel_notes/release_20_05.rst
@@ -81,6 +81,11 @@ New Features
   by making use of the event device capabilities. The event mode currently supports
   only inline IPsec protocol offload.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the device by appending the VF token.
 
 Removed Items
 -------------
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..efb64e2ba 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return 0;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1)) {
+		RTE_LOG(ERR, EAL, "The VF token length is too short\n");
+		return -1;
+	}
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',') {
+		RTE_LOG(ERR, EAL,
+			"The VF token ends with a invalid character : %c\n", c);
+		return -1;
+	}
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uuid)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is invalid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return -1;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* This VF token will be treated as a invalid device argument if the
+	 * PMD calls the rte_devargs parse API with its own valid argument list,
+	 * so it needs to purge this vfio-pci specific argument.
+	 */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+
+	return 0;
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 80dc9aa78..86d5a5f49 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..e5476ec6d 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,30 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index 4502aefed..916082b5d 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.1

Re: [dpdk-dev] [PATCH v8 2/2] eal: support for VFIO-PCI VF token

[Thomas Monjalon]

Hi,

A general comment, please try to limit line length (80 is a good limit),
and prefer breaking lines after a punctuation sign.
Try shortening sentences as well.

I would love seeing some documentation reviews from english native speakers.

18/04/2020 19:30, Haiyue Wang:
> --- a/doc/guides/linux_gsg/linux_drivers.rst
> +++ b/doc/guides/linux_gsg/linux_drivers.rst
> +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual functions, this feature is disabled
> +by default. When enabled, the PF needs a shared VF token (UUID) to setup the trust between SR-IOV PF and VFs. The VF
> +token is any kind of valid UUID value selected by the user. When the PF device is bound to ``vfio-pci`` module, it should
> +not have any VFs created, this is consistent as before for security reason.
> +
> +Some use cases about how to use the VF token:
> +
> + - The user just uses PF only for DPDK, then no VF token is required to start the PF device.

This use case is not using the token, so it should be mentioned out of this list.

> +
> + - The user wants to creat SR-IOV VFs on the PF device which is bound to ``vfio-pci`` module, then the user needs to select

creat -> create

> +   a valid UUID type VF token to start the PF device; after the VFs are created, this VF token is also required to access each
> +   VF device.
> +
> + - If the DPDK application that runs on PF device exits, and the user wants to start it with another different VF token
> +   value, it will be OK if no application (DPDP or KVM) runs on VF, otherwise, it will fail to start with a kernel message

DPDP -> DPDK.

> +   "[19145.688094] vfio-pci 0000:87:00.0: Incorrect VF token provided for device" shown. When all of the VFs are free, the

The timestamp is useless in the log.
Please insert the log after :: as a fixed-size font line.

> +   user can select a new VF token to start the PF device.
> +
> +The VFs created are bound to ``vfio-pci`` module automatically. DPDK will use the keyword ``vf_token`` as the device argument
> +to pass the VF token value to PF and its related VFs, the PMD should not use it, and this argument will be pruned from the
> +device argument list, so the PMD can parse its own valid device arguments successfully without seeing it.
> +
> +.. code-block:: console
> +
> +    1. sudo modprobe vfio-pci enable_sriov=1
> +
> +    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
> +
> +    3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
> +
> +    4. Start the PF:
> +        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i

How did you generate this token?

> +
> +    5. Start the VF:
> +        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf0 -- -i

Please do not make bullets and comments part of code-block.

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[David Marchand]

Hello,

On Sat, Apr 18, 2020 at 7:36 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
>
> v8: Update the document.

Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle
the ABI failure on rte_vfio_setup_device.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Wang, Haiyue]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Tuesday, April 21, 2020 00:53
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev <dev@dpdk.org>; Burakov, Anatoly <anatoly.burakov@intel.com>; Thomas Monjalon
> <thomas@monjalon.net>; Vamsi Attunuru <vattunuru@marvell.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Alex Williamson <alex.williamson@redhat.com>
> Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface
> 
> Hello,
> 
> On Sat, Apr 18, 2020 at 7:36 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> >
> > v8: Update the document.
> 
> Thanks for the last version.
> I had a look at the CI, I can see we are still missing bits to handle
> the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
is the best way to handle ABI issue.

> 
> 
> --
> David Marchand

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Thomas Monjalon]

20/04/2020 19:02, Wang, Haiyue:
> From: David Marchand <david.marchand@redhat.com>
> > 
> > Hello,
> > 
> > On Sat, Apr 18, 2020 at 7:36 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> > >
> > > v8: Update the document.
> > 
> > Thanks for the last version.
> > I had a look at the CI, I can see we are still missing bits to handle
> > the ABI failure on rte_vfio_setup_device.
> 
> Yes, not handle it now.
> 
> If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
> is the best way to handle ABI issue.

Please could you help finishing integration of __rte_internal?

This patch is blocked for now.

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Wang, Haiyue]

+Neil,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, April 21, 2020 01:13
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: David Marchand <david.marchand@redhat.com>; dev <dev@dpdk.org>; Burakov, Anatoly
> <anatoly.burakov@intel.com>; Vamsi Attunuru <vattunuru@marvell.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Alex Williamson <alex.williamson@redhat.com>
> Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface
> 
> 20/04/2020 19:02, Wang, Haiyue:
> > From: David Marchand <david.marchand@redhat.com>
> > >
> > > Hello,
> > >
> > > On Sat, Apr 18, 2020 at 7:36 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
> > > >
> > > > v8: Update the document.
> > >
> > > Thanks for the last version.
> > > I had a look at the CI, I can see we are still missing bits to handle
> > > the ABI failure on rte_vfio_setup_device.
> >
> > Yes, not handle it now.
> >
> > If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
> > is the best way to handle ABI issue.
> 
> Please could you help finishing integration of __rte_internal?
> 

I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

> This patch is blocked for now.
> 

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Thomas Monjalon]

20/04/2020 19:37, Wang, Haiyue:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 20/04/2020 19:02, Wang, Haiyue:
> > > From: David Marchand <david.marchand@redhat.com>
> > > > I had a look at the CI, I can see we are still missing bits to handle
> > > > the ABI failure on rte_vfio_setup_device.
> > >
> > > Yes, not handle it now.
> > >
> > > If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
> > > is the best way to handle ABI issue.
> > 
> > Please could you help finishing integration of __rte_internal?
> 
> I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
> http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Wang, Haiyue]

Hi Thomas & David,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, April 21, 2020 01:43
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: Neil Horman <nhorman@tuxdriver.com>; David Marchand <david.marchand@redhat.com>; dev
> <dev@dpdk.org>; Burakov, Anatoly <anatoly.burakov@intel.com>; Vamsi Attunuru <vattunuru@marvell.com>;
> Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Alex Williamson <alex.williamson@redhat.com>
> Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface
> 
> 20/04/2020 19:37, Wang, Haiyue:
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 20/04/2020 19:02, Wang, Haiyue:
> > > > From: David Marchand <david.marchand@redhat.com>
> > > > > I had a look at the CI, I can see we are still missing bits to handle
> > > > > the ABI failure on rte_vfio_setup_device.
> > > >
> > > > Yes, not handle it now.
> > > >
> > > > If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
> > > > is the best way to handle ABI issue.
> > >
> > > Please could you help finishing integration of __rte_internal?
> >
> > I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
> > http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/
> 
> It did not happen after several months.
> If you want to unblock your patches, I think it is safer to finish yourself.
> 

Unfortunately, this __rte_internal will still make the ci fail to run when move the function
to INTERNAL session:

--- a/devtools/libabigail.abignore
+++ b/devtools/libabigail.abignore
@@ -3,6 +3,11 @@
 [suppress_variable]
         symbol_version = EXPERIMENTAL

+[suppress_function]
+        symbol_version = INTERNAL
+[suppress_variable]
+        symbol_version = INTERNAL
+


--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -213,7 +213,6 @@ DPDK_20.0 {
        rte_vfio_is_enabled;
        rte_vfio_noiommu_is_enabled;
        rte_vfio_release_device;
-       rte_vfio_setup_device;
        rte_vlog;
        rte_zmalloc;
        rte_zmalloc_socket;
@@ -339,3 +338,10 @@ EXPERIMENTAL {
        # added in 20.05
        rte_log_can_log;
 };
+
+INTERNAL {
+       global:
+
+       # added in 20.05
+       rte_vfio_setup_device;
+};


Functions changes summary: 1 Removed, 0 Changed, 0 Added function
Variables changes summary: 0 Removed, 0 Changed, 0 Added variable

1 Removed function:

  [D] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)'    {rte_vfio_setup_device@@DPDK_20.0}

Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Thomas Monjalon]

21/04/2020 03:38, Wang, Haiyue:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 20/04/2020 19:37, Wang, Haiyue:
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > 20/04/2020 19:02, Wang, Haiyue:
> > > > > From: David Marchand <david.marchand@redhat.com>
> > > > > > I had a look at the CI, I can see we are still missing bits to handle
> > > > > > the ABI failure on rte_vfio_setup_device.
> > > > >
> > > > > Yes, not handle it now.
> > > > >
> > > > > If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
> > > > > is the best way to handle ABI issue.
> > > >
> > > > Please could you help finishing integration of __rte_internal?
> > >
> > > I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
> > > http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/
> > 
> > It did not happen after several months.
> > If you want to unblock your patches, I think it is safer to finish yourself.
> > 
> 
> Unfortunately, this __rte_internal will still make the ci fail to run when move the function
> to INTERNAL session:
[...]
> +INTERNAL {
> +       global:
> +
> +       # added in 20.05
> +       rte_vfio_setup_device;
> +};

Why do you need to move the symbol explicitly in .map?

The tool should ignore symbols moving to internal, as an exception until 20.11.

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Wang, Haiyue]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, April 21, 2020 10:13
> To: David Marchand <david.marchand@redhat.com>; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: Neil Horman <nhorman@tuxdriver.com>; dev <dev@dpdk.org>; Burakov, Anatoly
> <anatoly.burakov@intel.com>; Vamsi Attunuru <vattunuru@marvell.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Yigit, Ferruh <ferruh.yigit@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; Kinsella, Ray <ray.kinsella@intel.com>
> Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface
> 
> 21/04/2020 03:38, Wang, Haiyue:
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 20/04/2020 19:37, Wang, Haiyue:
> > > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > > 20/04/2020 19:02, Wang, Haiyue:
> > > > > > From: David Marchand <david.marchand@redhat.com>
> > > > > > > I had a look at the CI, I can see we are still missing bits to handle
> > > > > > > the ABI failure on rte_vfio_setup_device.
> > > > > >
> > > > > > Yes, not handle it now.
> > > > > >
> > > > > > If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
> > > > > > is the best way to handle ABI issue.
> > > > >
> > > > > Please could you help finishing integration of __rte_internal?
> > > >
> > > > I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
> > > > http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/
> > >
> > > It did not happen after several months.
> > > If you want to unblock your patches, I think it is safer to finish yourself.
> > >
> >
> > Unfortunately, this __rte_internal will still make the ci fail to run when move the function
> > to INTERNAL session:
> [...]
> > +INTERNAL {
> > +       global:
> > +
> > +       # added in 20.05
> > +       rte_vfio_setup_device;
> > +};
> 
> Why do you need to move the symbol explicitly in .map?
> The tool should ignore symbols moving to internal, as an exception until 20.11.
>

If not move the symbol explicitly in .map, another kind of error happened.

./devtools/check-abi.sh old_abi new_abi
Functions changes summary: 0 Removed, 1 Changed, 0 Added function
Variables changes summary: 0 Removed, 0 Changed, 0 Added variable

1 function with some indirect sub-type change:

  [C] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at eal_vfio.c:704:1 has some indirect sub-type changes:
    parameter 5 of type 'unsigned char*' was added

Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'
 

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Thomas Monjalon]

21/04/2020 04:52, Wang, Haiyue:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 21/04/2020 03:38, Wang, Haiyue:
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > 20/04/2020 19:37, Wang, Haiyue:
> > > > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > > > 20/04/2020 19:02, Wang, Haiyue:
> > > > > > > From: David Marchand <david.marchand@redhat.com>
> > > > > > > > I had a look at the CI, I can see we are still missing bits to handle
> > > > > > > > the ABI failure on rte_vfio_setup_device.
> > > > > > >
> > > > > > > Yes, not handle it now.
> > > > > > >
> > > > > > > If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
> > > > > > > is the best way to handle ABI issue.
> > > > > >
> > > > > > Please could you help finishing integration of __rte_internal?
> > > > >
> > > > > I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
> > > > > http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/
> > > >
> > > > It did not happen after several months.
> > > > If you want to unblock your patches, I think it is safer to finish yourself.
> > > >
> > >
> > > Unfortunately, this __rte_internal will still make the ci fail to run when move the function
> > > to INTERNAL session:
> > [...]
> > > +INTERNAL {
> > > +       global:
> > > +
> > > +       # added in 20.05
> > > +       rte_vfio_setup_device;
> > > +};
> > 
> > Why do you need to move the symbol explicitly in .map?
> > The tool should ignore symbols moving to internal, as an exception until 20.11.
> 
> If not move the symbol explicitly in .map, another kind of error happened.
> 
> ./devtools/check-abi.sh old_abi new_abi
> Functions changes summary: 0 Removed, 1 Changed, 0 Added function
> Variables changes summary: 0 Removed, 0 Changed, 0 Added variable
> 
> 1 function with some indirect sub-type change:
> 
>   [C] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at eal_vfio.c:704:1 has some indirect sub-type changes:
>     parameter 5 of type 'unsigned char*' was added
> 
> Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'

This is what I said: you need to add a rule to ignore internal symbols.

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

[Wang, Haiyue]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, April 21, 2020 16:48
> To: David Marchand <david.marchand@redhat.com>; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: Neil Horman <nhorman@tuxdriver.com>; dev <dev@dpdk.org>; Burakov, Anatoly
> <anatoly.burakov@intel.com>; Vamsi Attunuru <vattunuru@marvell.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Yigit, Ferruh <ferruh.yigit@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; Kinsella, Ray <ray.kinsella@intel.com>
> Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface
> 
> 21/04/2020 04:52, Wang, Haiyue:
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 21/04/2020 03:38, Wang, Haiyue:
> > > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > > 20/04/2020 19:37, Wang, Haiyue:
> > > > > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > > > > 20/04/2020 19:02, Wang, Haiyue:
> > > > > > > > From: David Marchand <david.marchand@redhat.com>
> > > > > > > > > I had a look at the CI, I can see we are still missing bits to handle
> > > > > > > > > the ABI failure on rte_vfio_setup_device.
> > > > > > > >
> > > > > > > > Yes, not handle it now.
> > > > > > > >
> > > > > > > > If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal
> > > > > > > > is the best way to handle ABI issue.
> > > > > > >
> > > > > > > Please could you help finishing integration of __rte_internal?
> > > > > >
> > > > > > I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
> > > > > >
> http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/
> > > > >
> > > > > It did not happen after several months.
> > > > > If you want to unblock your patches, I think it is safer to finish yourself.
> > > > >
> > > >
> > > > Unfortunately, this __rte_internal will still make the ci fail to run when move the function
> > > > to INTERNAL session:
> > > [...]
> > > > +INTERNAL {
> > > > +       global:
> > > > +
> > > > +       # added in 20.05
> > > > +       rte_vfio_setup_device;
> > > > +};
> > >
> > > Why do you need to move the symbol explicitly in .map?
> > > The tool should ignore symbols moving to internal, as an exception until 20.11.
> >
> > If not move the symbol explicitly in .map, another kind of error happened.
> >
> > ./devtools/check-abi.sh old_abi new_abi
> > Functions changes summary: 0 Removed, 1 Changed, 0 Added function
> > Variables changes summary: 0 Removed, 0 Changed, 0 Added variable
> >
> > 1 function with some indirect sub-type change:
> >
> >   [C] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at
> eal_vfio.c:704:1 has some indirect sub-type changes:
> >     parameter 5 of type 'unsigned char*' was added
> >
> > Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --
> headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump
> new_abi/dump/librte_eal.dump'
> 
> This is what I said: you need to add a rule to ignore internal symbols.
> 

Got the kind reply from Dodji, the maintainer of libabigail, his suggestion
meets what we have done for 'EXPERIMENTAL'. So it seems that have to mark
all needed functions as INTERNAL firstly, which obviously will break the CI
as the public functions are removed....


"
__PROJECT_INTERNAL_USE_ONLY_VERSION__ {
  global: funA
};
...
Then, once you have all your internal functions marked in the ELF binary
with the proper ELF version string, you can tell libabigail to suppress
all functions that have that version by writing a suppression
specification file that has this content:


[suppress_function]
  symbol_version = __PROJECT_INTERNAL_USE_ONLY_VERSION__
"

[dpdk-dev] [PATCH v9 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v9: Rewrite the document.

v8: Update the document.
    https://patchwork.dpdk.org/cover/68859/

v7: Add the Fixes tag in uuid, the release note and help
    document.
    https://patchwork.dpdk.org/cover/68845/

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.
    https://patchwork.dpdk.org/cover/68367/

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.
    https://patchwork.dpdk.org/cover/68364/

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.
    https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
    https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.
         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.
        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/ 
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 devtools/libabigail.abignore           |  2 +
 doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_uuid.h      |  2 +
 lib/librte_eal/include/rte_vfio.h      | 24 ++++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 8 files changed, 163 insertions(+), 8 deletions(-)

-- 
2.26.2

[dpdk-dev] [PATCH v9 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.2

[dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
Tested-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 devtools/libabigail.abignore           |  2 +
 doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_vfio.h      | 24 ++++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 7 files changed, 161 insertions(+), 8 deletions(-)

diff --git a/devtools/libabigail.abignore b/devtools/libabigail.abignore
index cd86d89ca..01d987a1e 100644
--- a/devtools/libabigail.abignore
+++ b/devtools/libabigail.abignore
@@ -6,6 +6,8 @@
 ; Explicit ignore for driver-only ABI
 [suppress_type]
         name = rte_cryptodev_ops
+[suppress_function]
+	name = rte_vfio_setup_device
 ; Ignore this enum update as it is part of an experimental API
 [suppress_type]
         type_kind = enum
diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index 238f3e900..b42fd708b 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -72,11 +72,50 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+Also if the DPDK application running on the PF device exits, the user wants to start
+the PF with another different VF token value, it has no issue if no application like
+DPDK or KVM runs on VFs. Otherwise, the PF will fail to start until all VFs are free
+to use, after that, the user can select a new VF token to start the PF device.
+
+DPDK will use the keyword ``vf_token`` as the device argument to pass the VF token
+value to PF and its related VFs, the PMD should not use it, and this argument will
+be pruned from the device argument list, so the PMD can parse its own valid device
+arguments successfully.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
+         -w 86:00.0,vf_token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
+         -w 86:02.0,vf_token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/rel_notes/release_20_05.rst b/doc/guides/rel_notes/release_20_05.rst
index 709372e5e..9460e1eb2 100644
--- a/doc/guides/rel_notes/release_20_05.rst
+++ b/doc/guides/rel_notes/release_20_05.rst
@@ -97,6 +97,11 @@ New Features
   by making use of the event device capabilities. The event mode currently supports
   only inline IPsec protocol offload.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the device by appending the VF token.
 
 Removed Items
 -------------
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..efb64e2ba 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return 0;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1)) {
+		RTE_LOG(ERR, EAL, "The VF token length is too short\n");
+		return -1;
+	}
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',') {
+		RTE_LOG(ERR, EAL,
+			"The VF token ends with a invalid character : %c\n", c);
+		return -1;
+	}
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uuid)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is invalid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return -1;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* This VF token will be treated as a invalid device argument if the
+	 * PMD calls the rte_devargs parse API with its own valid argument list,
+	 * so it needs to purge this vfio-pci specific argument.
+	 */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+
+	return 0;
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 80dc9aa78..86d5a5f49 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..28d918cde 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,33 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
+ *   The vf_token can be zero uuid, which will be ignored to pass into the
+ *   vfio-pci module.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index d26e1649a..e8d7cbda5 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2083,7 +2096,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
-- 
2.26.2

Re: [dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token

[Thomas Monjalon]

22/04/2020 07:08, Haiyue Wang:
> --- a/devtools/libabigail.abignore
> +++ b/devtools/libabigail.abignore
> +[suppress_function]
> +	name = rte_vfio_setup_device

There is no comment to justify ignoring this symbol.

The current policy is to reject any change in libabigail.abignore
which could be better handled with (unfinished) rte_internal mark.

Re: [dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token

[Wang, Haiyue]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, April 22, 2020 16:24
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>; vattunuru@marvell.com;
> jerinj@marvell.com; alex.williamson@redhat.com; david.marchand@redhat.com
> Subject: Re: [PATCH v9 2/2] eal: support for VFIO-PCI VF token
> 
> 22/04/2020 07:08, Haiyue Wang:
> > --- a/devtools/libabigail.abignore
> > +++ b/devtools/libabigail.abignore
> > +[suppress_function]
> > +	name = rte_vfio_setup_device
> 
> There is no comment to justify ignoring this symbol.
> 
> The current policy is to reject any change in libabigail.abignore
> which could be better handled with (unfinished) rte_internal mark.
> 
> 

While updating the vfio patch, also doing the ABI things. Found that:
1). [suppress_function]
	name = rte_vfio_setup_device
    can make CI happy, but not scale.

2). rte_internal is beautiful, but make CI cry : https://patchwork.dpdk.org/patch/69109/
1 Removed function:

  [D] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)'    {rte_vfio_setup_device@@DPDK_20.0}

[dpdk-dev] [PATCH v10 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  6 +++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_uuid.h      |  2 +
 lib/librte_eal/include/rte_vfio.h      | 26 ++++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 lib/librte_eal/rte_eal_version.map     |  8 ++-
 8 files changed, 171 insertions(+), 9 deletions(-)

-- 
2.26.2

[dpdk-dev] [PATCH v10 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.2

[dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
Tested-by: Vamsi Attunuru <vattunuru@marvell.com>
---
 doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  6 +++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_vfio.h      | 26 ++++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 lib/librte_eal/rte_eal_version.map     |  8 ++-
 7 files changed, 169 insertions(+), 9 deletions(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index 238f3e900..b42fd708b 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -72,11 +72,50 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+Also if the DPDK application running on the PF device exits, the user wants to start
+the PF with another different VF token value, it has no issue if no application like
+DPDK or KVM runs on VFs. Otherwise, the PF will fail to start until all VFs are free
+to use, after that, the user can select a new VF token to start the PF device.
+
+DPDK will use the keyword ``vf_token`` as the device argument to pass the VF token
+value to PF and its related VFs, the PMD should not use it, and this argument will
+be pruned from the device argument list, so the PMD can parse its own valid device
+arguments successfully.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
+         -w 86:00.0,vf_token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
+         -w 86:02.0,vf_token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/rel_notes/release_20_05.rst b/doc/guides/rel_notes/release_20_05.rst
index b124c3f28..722c61e67 100644
--- a/doc/guides/rel_notes/release_20_05.rst
+++ b/doc/guides/rel_notes/release_20_05.rst
@@ -93,6 +93,12 @@ New Features
   * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
     contexts from the port.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
+
 * **Updated Amazon ena driver.**
 
   Updated ena PMD with new features and improvements, including:
diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
index 64cd84a68..efb64e2ba 100644
--- a/drivers/bus/pci/linux/pci_vfio.c
+++ b/drivers/bus/pci/linux/pci_vfio.c
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return 0;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1)) {
+		RTE_LOG(ERR, EAL, "The VF token length is too short\n");
+		return -1;
+	}
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',') {
+		RTE_LOG(ERR, EAL,
+			"The VF token ends with a invalid character : %c\n", c);
+		return -1;
+	}
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uuid)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is invalid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return -1;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* This VF token will be treated as a invalid device argument if the
+	 * PMD calls the rte_devargs parse API with its own valid argument list,
+	 * so it needs to purge this vfio-pci specific argument.
+	 */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+
+	return 0;
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 540b7d38c..e42d3e12e 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -1005,7 +1005,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/include/rte_vfio.h b/lib/librte_eal/include/rte_vfio.h
index 20ed8c45a..430f98770 100644
--- a/lib/librte_eal/include/rte_vfio.h
+++ b/lib/librte_eal/include/rte_vfio.h
@@ -16,6 +16,9 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_compat.h>
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +105,34 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
+ *   The vf_token can be zero uuid, which will be ignored to pass into the
+ *   vfio-pci module.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
+__rte_internal
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index d26e1649a..e8d7cbda5 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2083,7 +2096,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 6088e7f6c..5bcdccae6 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -213,7 +213,6 @@ DPDK_20.0 {
 	rte_vfio_is_enabled;
 	rte_vfio_noiommu_is_enabled;
 	rte_vfio_release_device;
-	rte_vfio_setup_device;
 	rte_vlog;
 	rte_zmalloc;
 	rte_zmalloc_socket;
@@ -388,3 +387,10 @@ EXPERIMENTAL {
 	rte_trace_regexp;
 	rte_trace_save;
 };
+
+INTERNAL {
+	global:
+
+	# added in 20.05
+	rte_vfio_setup_device;
+};
-- 
2.26.2

Re: [dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token

[Burakov, Anatoly]

On 26-Apr-20 2:55 AM, Haiyue Wang wrote:
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support since 5.7.
> 
> The VF token can be set by a vfio-pci based PF driver and must be known
> by the vfio-pci based VF driver in order to gain access to the device.
> 
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
> Tested-by: Vamsi Attunuru <vattunuru@marvell.com>
> ---

LGTM

Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

-- 
Thanks,
Anatoly

Re: [dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token

[Andrew Rybchenko]

On 4/26/20 4:55 AM, Haiyue Wang wrote:
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support since 5.7.
> 
> The VF token can be set by a vfio-pci based PF driver and must be known
> by the vfio-pci based VF driver in order to gain access to the device.
> 
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> Acked-by: Vamsi Attunuru <vattunuru@marvell.com>
> Tested-by: Vamsi Attunuru <vattunuru@marvell.com>

Few points below, anyway:
Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>

[snip]

> diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
> index 238f3e900..b42fd708b 100644
> --- a/doc/guides/linux_gsg/linux_drivers.rst
> +++ b/doc/guides/linux_gsg/linux_drivers.rst
> @@ -72,11 +72,50 @@ Note that in order to use VFIO, your kernel must support it.
>  VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
>  however please consult your distributions documentation to make sure that is the case.
>  
> +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
> +functions. After the PF is bound to vfio-pci module, the user can create the VFs
> +by sysfs interface, and these VFs are bound to vfio-pci module automatically.
> +
> +When the PF is bound to vfio-pci, it has initial VF token generated by random. For
> +security reason, this token is write only, the user can't read it from the kernel
> +directly. For accessing the VF, the user needs to start the PF with token parameter
> +to setup a VF token (uuid format), then the VF can be accessed with this new known
> +VF token.

If token is write-only in kernel sysfs, shouldn't we make it
invisible in ps output (procfs)? I.e. substitute with something
like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
However, it does not look easy. Just a thought.

[snip]

> diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c
> index 64cd84a68..efb64e2ba 100644
> --- a/drivers/bus/pci/linux/pci_vfio.c
> +++ b/drivers/bus/pci/linux/pci_vfio.c
> @@ -11,6 +11,7 @@
>  #include <sys/mman.h>
>  #include <stdbool.h>
>  
> +#include <rte_devargs.h>
>  #include <rte_log.h>
>  #include <rte_pci.h>
>  #include <rte_bus_pci.h>
> @@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
>  	return ret;
>  }
>  
> +static int
> +vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
> +{
> +#define VF_TOKEN_ARG "vf_token="
> +	char c, *p, *vf_token;
> +
> +	memset(uuid, 0, sizeof(rte_uuid_t));
> +
> +	if (devargs == NULL)
> +		return 0;
> +
> +	p = strstr(devargs->args, VF_TOKEN_ARG);
> +	if (!p)

Please, p == NULL as you do few lines above.

[snip]

[dpdk-dev] [PATCH v11 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v11: Use the eal parameter to pass the VF token, then not every PCI
     device needs to be specified with this token. Also no ABI issue
     now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 16 ++++++++-
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 114 insertions(+), 2 deletions(-)

-- 
2.26.2

[dpdk-dev] [PATCH v11 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.2

[dpdk-dev] [PATCH v11 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 16 ++++++++-
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index 238f3e900..910397243 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/linux_gsg/linux_eal_parameters.rst b/doc/guides/linux_gsg/linux_eal_parameters.rst
index b2cc60e44..bd3977cb3 100644
--- a/doc/guides/linux_gsg/linux_eal_parameters.rst
+++ b/doc/guides/linux_gsg/linux_eal_parameters.rst
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/rel_notes/release_20_05.rst b/doc/guides/rel_notes/release_20_05.rst
index ab20a7d02..c81deb06c 100644
--- a/doc/guides/rel_notes/release_20_05.rst
+++ b/doc/guides/rel_notes/release_20_05.rst
@@ -93,6 +93,12 @@ New Features
   * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
     contexts from the port.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
+
 * **Updated Amazon ena driver.**
 
   Updated ena PMD with new features and improvements, including:
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 418731ca4..dbce919ad 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -89,6 +89,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index a42f34923..28154f0ef 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 90ead1b7c..cd5fd01f2 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -67,6 +67,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 540b7d38c..00b02855e 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	memset(vf_token, 0, sizeof(rte_uuid_t));
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
index 2f9ed298d..d9040ce14 100644
--- a/lib/librte_eal/include/rte_eal.h
+++ b/lib/librte_eal/include/rte_eal.h
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * The user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index aa72d3650..05cd6e497 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index d26e1649a..e8a264bea 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -712,6 +712,8 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,20 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_vf_token(vf_token);
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 6088e7f6c..5b7dd3e81 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -373,6 +373,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
-- 
2.26.2

[dpdk-dev] [PATCH v12 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
     device needs to be specified with this token. Also no ABI issue
     now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 118 insertions(+), 1 deletion(-)

-- 
2.26.2

[dpdk-dev] [PATCH v12 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.2

[dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 116 insertions(+), 1 deletion(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index 238f3e900..910397243 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/linux_gsg/linux_eal_parameters.rst b/doc/guides/linux_gsg/linux_eal_parameters.rst
index b2cc60e44..bd3977cb3 100644
--- a/doc/guides/linux_gsg/linux_eal_parameters.rst
+++ b/doc/guides/linux_gsg/linux_eal_parameters.rst
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/rel_notes/release_20_05.rst b/doc/guides/rel_notes/release_20_05.rst
index ab20a7d02..c81deb06c 100644
--- a/doc/guides/rel_notes/release_20_05.rst
+++ b/doc/guides/rel_notes/release_20_05.rst
@@ -93,6 +93,12 @@ New Features
   * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
     contexts from the port.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
+
 * **Updated Amazon ena driver.**
 
   Updated ena PMD with new features and improvements, including:
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 418731ca4..dbce919ad 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -89,6 +89,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index a42f34923..28154f0ef 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 90ead1b7c..cd5fd01f2 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -67,6 +67,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 540b7d38c..00b02855e 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	memset(vf_token, 0, sizeof(rte_uuid_t));
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
index 2f9ed298d..d9040ce14 100644
--- a/lib/librte_eal/include/rte_eal.h
+++ b/lib/librte_eal/include/rte_eal.h
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * The user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index aa72d3650..05cd6e497 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index d26e1649a..6b8496b52 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 6088e7f6c..5b7dd3e81 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -373,6 +373,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
-- 
2.26.2

Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

[Burakov, Anatoly]

On 05-May-20 11:34 AM, Haiyue Wang wrote:
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support since 5.7.
> 
> The VF token can be set by a vfio-pci based PF driver and must be known
> by the vfio-pci based VF driver in order to gain access to the device.
> 
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> ---

<snip>

> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> index 540b7d38c..00b02855e 100644
> --- a/lib/librte_eal/freebsd/eal.c
> +++ b/lib/librte_eal/freebsd/eal.c
> @@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
>   	return RTE_INTR_MODE_NONE;
>   }
>   
> +void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
> +{
> +	memset(vf_token, 0, sizeof(rte_uuid_t));
> +}

What's the purpose of memset(0) here? Presumably, if the API is not 
supposed to be supported, the function should have no effect?

> +
>   int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
>   		      __rte_unused const char *dev_addr,
>   		      __rte_unused int *vfio_dev_fd,
> diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
> index 2f9ed298d..d9040ce14 100644
> --- a/lib/librte_eal/include/rte_eal.h
> +++ b/lib/librte_eal/include/rte_eal.h
> @@ -21,6 +21,7 @@
>   #include <rte_bus.h>
>   
>   #include <rte_pci_dev_feature_defs.h>
> +#include <rte_uuid.h>
>   
>   #ifdef __cplusplus
>   extern "C" {
> @@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
>    */
>   enum rte_intr_mode rte_eal_vfio_intr_mode(void);
>   
> +
> +/**
> + * The user-configured vfio VF token.

The description is unclear. Suggested rewording:

Copy the user-configured vfio VF token.

> + *
> + * @param vf_token
> + *   vfio VF token configured with the command line is copied
> + *   into this parameter, zero uuid by default.
> + */
> +__rte_experimental
> +void rte_eal_vfio_vf_token(rte_uuid_t vf_token);

Maybe rte_eal_vfio_get_vf_token()? I imagine this would be a more 
descriptive name for this API.

Once all of the above is addressed (or explained),

Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

-- 
Thanks,
Anatoly

Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

[Wang, Haiyue]

Hi Anatoly,

> -----Original Message-----
> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> Sent: Wednesday, May 6, 2020 18:27
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; jerinj@marvell.com;
> david.marchand@redhat.com
> Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token
> 
> On 05-May-20 11:34 AM, Haiyue Wang wrote:
> > The kernel module vfio-pci introduces the VF token to enable SR-IOV
> > support since 5.7.
> >
> > The VF token can be set by a vfio-pci based PF driver and must be known
> > by the vfio-pci based VF driver in order to gain access to the device.
> >
> > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > ---
> 
> <snip>
> 
> > diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> > index 540b7d38c..00b02855e 100644
> > --- a/lib/librte_eal/freebsd/eal.c
> > +++ b/lib/librte_eal/freebsd/eal.c
> > @@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
> >   	return RTE_INTR_MODE_NONE;
> >   }
> >
> > +void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
> > +{
> > +	memset(vf_token, 0, sizeof(rte_uuid_t));
> > +}
> 
> What's the purpose of memset(0) here? Presumably, if the API is not
> supposed to be supported, the function should have no effect?
> 

Yes, originally I meant to no parse the input, then return zero UUID.
I changed it to the same as Linux: 
	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
Since it is inited to zero.

> > +
> >   int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
> >   		      __rte_unused const char *dev_addr,
> >   		      __rte_unused int *vfio_dev_fd,
> > diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
> > index 2f9ed298d..d9040ce14 100644
> > --- a/lib/librte_eal/include/rte_eal.h
> > +++ b/lib/librte_eal/include/rte_eal.h
> > @@ -21,6 +21,7 @@
> >   #include <rte_bus.h>
> >
> >   #include <rte_pci_dev_feature_defs.h>
> > +#include <rte_uuid.h>
> >
> >   #ifdef __cplusplus
> >   extern "C" {
> > @@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
> >    */
> >   enum rte_intr_mode rte_eal_vfio_intr_mode(void);
> >
> > +
> > +/**
> > + * The user-configured vfio VF token.
> 
> The description is unclear. Suggested rewording:
> 
> Copy the user-configured vfio VF token.
> 

+1.

> > + *
> > + * @param vf_token
> > + *   vfio VF token configured with the command line is copied
> > + *   into this parameter, zero uuid by default.
> > + */
> > +__rte_experimental
> > +void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
> 
> Maybe rte_eal_vfio_get_vf_token()? I imagine this would be a more
> descriptive name for this API.
> 

Yes, this looks better. Fixed.

> Once all of the above is addressed (or explained),
> 
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> 
> --
> Thanks,
> Anatoly

Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

[Burakov, Anatoly]

On 06-May-20 12:35 PM, Wang, Haiyue wrote:
> Hi Anatoly,
> 
>> -----Original Message-----
>> From: Burakov, Anatoly <anatoly.burakov@intel.com>
>> Sent: Wednesday, May 6, 2020 18:27
>> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; jerinj@marvell.com;
>> david.marchand@redhat.com
>> Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token
>>
>> On 05-May-20 11:34 AM, Haiyue Wang wrote:
>>> The kernel module vfio-pci introduces the VF token to enable SR-IOV
>>> support since 5.7.
>>>
>>> The VF token can be set by a vfio-pci based PF driver and must be known
>>> by the vfio-pci based VF driver in order to gain access to the device.
>>>
>>> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
>>> ---
>>
>> <snip>
>>
>>> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
>>> index 540b7d38c..00b02855e 100644
>>> --- a/lib/librte_eal/freebsd/eal.c
>>> +++ b/lib/librte_eal/freebsd/eal.c
>>> @@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
>>>    	return RTE_INTR_MODE_NONE;
>>>    }
>>>
>>> +void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
>>> +{
>>> +	memset(vf_token, 0, sizeof(rte_uuid_t));
>>> +}
>>
>> What's the purpose of memset(0) here? Presumably, if the API is not
>> supposed to be supported, the function should have no effect?
>>
> 
> Yes, originally I meant to no parse the input, then return zero UUID.
> I changed it to the same as Linux:
> 	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
> Since it is inited to zero.

Yes but on Linux it has effect because Linux supports it. On FreeBSD 
there's no VFIO, so the function should IMO have no effect. However, i 
don't have a strong opinion on this, so i'm fine either way.

-- 
Thanks,
Anatoly

Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

[Wang, Haiyue]

> -----Original Message-----
> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> Sent: Wednesday, May 6, 2020 19:40
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; jerinj@marvell.com;
> david.marchand@redhat.com
> Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token
> 
> On 06-May-20 12:35 PM, Wang, Haiyue wrote:
> > Hi Anatoly,
> >
> >> -----Original Message-----
> >> From: Burakov, Anatoly <anatoly.burakov@intel.com>
> >> Sent: Wednesday, May 6, 2020 18:27
> >> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; thomas@monjalon.net; jerinj@marvell.com;
> >> david.marchand@redhat.com
> >> Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token
> >>
> >> On 05-May-20 11:34 AM, Haiyue Wang wrote:
> >>> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> >>> support since 5.7.
> >>>
> >>> The VF token can be set by a vfio-pci based PF driver and must be known
> >>> by the vfio-pci based VF driver in order to gain access to the device.
> >>>
> >>> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> >>> ---
> >>
> >> <snip>
> >>
> >>> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> >>> index 540b7d38c..00b02855e 100644
> >>> --- a/lib/librte_eal/freebsd/eal.c
> >>> +++ b/lib/librte_eal/freebsd/eal.c
> >>> @@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
> >>>    	return RTE_INTR_MODE_NONE;
> >>>    }
> >>>
> >>> +void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
> >>> +{
> >>> +	memset(vf_token, 0, sizeof(rte_uuid_t));
> >>> +}
> >>
> >> What's the purpose of memset(0) here? Presumably, if the API is not
> >> supposed to be supported, the function should have no effect?
> >>
> >
> > Yes, originally I meant to no parse the input, then return zero UUID.
> > I changed it to the same as Linux:
> > 	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
> > Since it is inited to zero.
> 
> Yes but on Linux it has effect because Linux supports it. On FreeBSD
> there's no VFIO, so the function should IMO have no effect. However, i
> don't have a strong opinion on this, so i'm fine either way.
> 

Uh, seems empty function should be good one. I will change it. ;-)

> --
> Thanks,
> Anatoly

[dpdk-dev] [PATCH v13 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
     device needs to be specified with this token. Also no ABI issue
     now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 117 insertions(+), 1 deletion(-)

-- 
2.26.2

[dpdk-dev] [PATCH v13 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.2

[dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 115 insertions(+), 1 deletion(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index 238f3e900..910397243 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/linux_gsg/linux_eal_parameters.rst b/doc/guides/linux_gsg/linux_eal_parameters.rst
index b2cc60e44..bd3977cb3 100644
--- a/doc/guides/linux_gsg/linux_eal_parameters.rst
+++ b/doc/guides/linux_gsg/linux_eal_parameters.rst
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/rel_notes/release_20_05.rst b/doc/guides/rel_notes/release_20_05.rst
index c287cb48a..b19b03f83 100644
--- a/doc/guides/rel_notes/release_20_05.rst
+++ b/doc/guides/rel_notes/release_20_05.rst
@@ -93,6 +93,12 @@ New Features
   * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
     contexts from the port.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
+
 * **Updated Amazon ena driver.**
 
   Updated ena PMD with new features and improvements, including:
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 418731ca4..dbce919ad 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -89,6 +89,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index a42f34923..28154f0ef 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 90ead1b7c..cd5fd01f2 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -67,6 +67,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 540b7d38c..07f17f182 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -1002,6 +1002,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
index 2f9ed298d..744370013 100644
--- a/lib/librte_eal/include/rte_eal.h
+++ b/lib/librte_eal/include/rte_eal.h
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index aa72d3650..f394c281c 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index d26e1649a..5b2f6b305 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 6088e7f6c..f7043af69 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -373,6 +373,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
-- 
2.26.2

Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

[Andrew Rybchenko]

On 5/6/20 2:35 PM, Haiyue Wang wrote:
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support since 5.7.
> 
> The VF token can be set by a vfio-pci based PF driver and must be known
> by the vfio-pci based VF driver in order to gain access to the device.
> 
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

Sorry, lost from my view new versions of the patch series.

Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>

> diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
> index 238f3e900..910397243 100644
> --- a/doc/guides/linux_gsg/linux_drivers.rst
> +++ b/doc/guides/linux_gsg/linux_drivers.rst
> @@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
>  VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
>  however please consult your distributions documentation to make sure that is the case.
>  
> +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
> +functions. After the PF is bound to vfio-pci module, the user can create the VFs
> +by sysfs interface, and these VFs are bound to vfio-pci module automatically.
> +
> +When the PF is bound to vfio-pci, it has initial VF token generated by random. For
> +security reason, this token is write only, the user can't read it from the kernel
> +directly. For accessing the VF, the user needs to start the PF with token parameter
> +to setup a VF token (uuid format), then the VF can be accessed with this new known
> +VF token.

If token is write-only in kernel sysfs, shouldn't we make it
invisible in ps output? I.e. substitute with something like
xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.

[snip]

Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

[Wang, Haiyue]

> -----Original Message-----
> From: Andrew Rybchenko <arybchenko@solarflare.com>
> Sent: Thursday, May 7, 2020 00:51
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>;
> thomas@monjalon.net; jerinj@marvell.com; david.marchand@redhat.com
> Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token
> 
> On 5/6/20 2:35 PM, Haiyue Wang wrote:
> > The kernel module vfio-pci introduces the VF token to enable SR-IOV
> > support since 5.7.
> >
> > The VF token can be set by a vfio-pci based PF driver and must be known
> > by the vfio-pci based VF driver in order to gain access to the device.
> >
> > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> 
> Sorry, lost from my view new versions of the patch series.
> 
> Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> 
> > diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
> > index 238f3e900..910397243 100644
> > --- a/doc/guides/linux_gsg/linux_drivers.rst
> > +++ b/doc/guides/linux_gsg/linux_drivers.rst
> > @@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
> >  VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually
> present by default,
> >  however please consult your distributions documentation to make sure that is the case.
> >
> > +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
> > +functions. After the PF is bound to vfio-pci module, the user can create the VFs
> > +by sysfs interface, and these VFs are bound to vfio-pci module automatically.
> > +
> > +When the PF is bound to vfio-pci, it has initial VF token generated by random. For
> > +security reason, this token is write only, the user can't read it from the kernel
> > +directly. For accessing the VF, the user needs to start the PF with token parameter
> > +to setup a VF token (uuid format), then the VF can be accessed with this new known
> > +VF token.
> 
> If token is write-only in kernel sysfs, shouldn't we make it
> invisible in ps output? I.e. substitute with something like
> xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
> It is a bit easier with the new design. Just a thought.
> 

In fact, no sysfs for VF token, just write-only IOCTL. ;-)

> [snip]

Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

[Andrew Rybchenko]

On 5/6/20 7:56 PM, Wang, Haiyue wrote:
>> -----Original Message-----
>> From: Andrew Rybchenko <arybchenko@solarflare.com>
>> Sent: Thursday, May 7, 2020 00:51
>> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>;
>> thomas@monjalon.net; jerinj@marvell.com; david.marchand@redhat.com
>> Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token
>>
>> On 5/6/20 2:35 PM, Haiyue Wang wrote:
>>> The kernel module vfio-pci introduces the VF token to enable SR-IOV
>>> support since 5.7.
>>>
>>> The VF token can be set by a vfio-pci based PF driver and must be known
>>> by the vfio-pci based VF driver in order to gain access to the device.
>>>
>>> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
>>> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
>>
>> Sorry, lost from my view new versions of the patch series.
>>
>> Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
>>
>>> diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
>>> index 238f3e900..910397243 100644
>>> --- a/doc/guides/linux_gsg/linux_drivers.rst
>>> +++ b/doc/guides/linux_gsg/linux_drivers.rst
>>> @@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
>>>  VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually
>> present by default,
>>>  however please consult your distributions documentation to make sure that is the case.
>>>
>>> +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
>>> +functions. After the PF is bound to vfio-pci module, the user can create the VFs
>>> +by sysfs interface, and these VFs are bound to vfio-pci module automatically.
>>> +
>>> +When the PF is bound to vfio-pci, it has initial VF token generated by random. For
>>> +security reason, this token is write only, the user can't read it from the kernel
>>> +directly. For accessing the VF, the user needs to start the PF with token parameter
>>> +to setup a VF token (uuid format), then the VF can be accessed with this new known
>>> +VF token.
>>
>> If token is write-only in kernel sysfs, shouldn't we make it
>> invisible in ps output? I.e. substitute with something like
>> xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
>> It is a bit easier with the new design. Just a thought.
>>
> 
> In fact, no sysfs for VF token, just write-only IOCTL. ;-)

OK, got it. The question remains anyway. Should it be treated
as a secret with at least minimal security precaution?

Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

[Wang, Haiyue]

+Alex

> -----Original Message-----
> From: Andrew Rybchenko <arybchenko@solarflare.com>
> Sent: Thursday, May 7, 2020 00:59
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>;
> thomas@monjalon.net; jerinj@marvell.com; david.marchand@redhat.com
> Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token
> 
> On 5/6/20 7:56 PM, Wang, Haiyue wrote:
> >> -----Original Message-----
> >> From: Andrew Rybchenko <arybchenko@solarflare.com>
> >> Sent: Thursday, May 7, 2020 00:51
> >> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org; Burakov, Anatoly
> <anatoly.burakov@intel.com>;
> >> thomas@monjalon.net; jerinj@marvell.com; david.marchand@redhat.com
> >> Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token
> >>
> >> On 5/6/20 2:35 PM, Haiyue Wang wrote:
> >>> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> >>> support since 5.7.
> >>>
> >>> The VF token can be set by a vfio-pci based PF driver and must be known
> >>> by the vfio-pci based VF driver in order to gain access to the device.
> >>>
> >>> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> >>> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> >>
> >> Sorry, lost from my view new versions of the patch series.
> >>
> >> Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> >>
> >>> diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
> >>> index 238f3e900..910397243 100644
> >>> --- a/doc/guides/linux_gsg/linux_drivers.rst
> >>> +++ b/doc/guides/linux_gsg/linux_drivers.rst
> >>> @@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
> >>>  VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually
> >> present by default,
> >>>  however please consult your distributions documentation to make sure that is the case.
> >>>
> >>> +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
> >>> +functions. After the PF is bound to vfio-pci module, the user can create the VFs
> >>> +by sysfs interface, and these VFs are bound to vfio-pci module automatically.
> >>> +
> >>> +When the PF is bound to vfio-pci, it has initial VF token generated by random. For
> >>> +security reason, this token is write only, the user can't read it from the kernel
> >>> +directly. For accessing the VF, the user needs to start the PF with token parameter
> >>> +to setup a VF token (uuid format), then the VF can be accessed with this new known
> >>> +VF token.
> >>
> >> If token is write-only in kernel sysfs, shouldn't we make it
> >> invisible in ps output? I.e. substitute with something like
> >> xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
> >> It is a bit easier with the new design. Just a thought.
> >>
> >
> > In fact, no sysfs for VF token, just write-only IOCTL. ;-)
> 
> OK, got it. The question remains anyway. Should it be treated
> as a secret with at least minimal security precaution?
>

Sounds yes, and also it looks like be more friendly for user to check whether
this PF/VF have a VF token required or not by cat /sys/...

@Alex may consider this design.

[dpdk-dev] [PATCH v14 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v14: Rebase the patch for 20.08 release note.

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
     device needs to be specified with this token. Also no ABI issue
     now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 116 insertions(+), 1 deletion(-)

-- 
2.26.2

[dpdk-dev] [PATCH v14 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.2

[dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 114 insertions(+), 1 deletion(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index d40b495c1..02fba7d1e 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/linux_gsg/linux_eal_parameters.rst b/doc/guides/linux_gsg/linux_eal_parameters.rst
index b2cc60e44..bd3977cb3 100644
--- a/doc/guides/linux_gsg/linux_eal_parameters.rst
+++ b/doc/guides/linux_gsg/linux_eal_parameters.rst
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/rel_notes/release_20_08.rst b/doc/guides/rel_notes/release_20_08.rst
index 39064afbe..dfce08b89 100644
--- a/doc/guides/rel_notes/release_20_08.rst
+++ b/doc/guides/rel_notes/release_20_08.rst
@@ -56,6 +56,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
 
 Removed Items
 -------------
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 8f2cbd1c6..e7b643a99 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -92,6 +92,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -297,6 +298,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index c650bc081..43095b7a2 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 18e6da9ab..89769d48b 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index c41f265fa..4ff5028d8 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
index 2f9ed298d..744370013 100644
--- a/lib/librte_eal/include/rte_eal.h
+++ b/lib/librte_eal/include/rte_eal.h
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index f162124a3..00f3fdced 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index d26e1649a..5b2f6b305 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index d8038749a..93a4d30f8 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -372,6 +372,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
-- 
2.26.2

Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token

[Ye Xiaolong]

On 05/28, Haiyue Wang wrote:
>The kernel module vfio-pci introduces the VF token to enable SR-IOV
>support since 5.7.
>
>The VF token can be set by a vfio-pci based PF driver and must be known
>by the vfio-pci based VF driver in order to gain access to the device.
>
>Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
>Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
>Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
>---

[snip]

>+
>+/**
>+ * Copy the user-configured vfio VF token.
>+ *
>+ * @param vf_token
>+ *   vfio VF token configured with the command line is copied
>+ *   into this parameter, zero uuid by default.
>+ */
>+__rte_experimental
>+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
>+

Minor nit, missing the EXPERIMENTAL warning int the comment.

 * @warning
 * @b EXPERIMENTAL: this API may change without prior notice
 *

Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token

[Wang, Haiyue]

> -----Original Message-----
> From: Ye, Xiaolong <xiaolong.ye@intel.com>
> Sent: Friday, May 29, 2020 09:26
> To: Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Burakov, Anatoly <anatoly.burakov@intel.com>; thomas@monjalon.net;
> jerinj@marvell.com; david.marchand@redhat.com; arybchenko@solarflare.com
> Subject: Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token
> 
> On 05/28, Haiyue Wang wrote:
> >The kernel module vfio-pci introduces the VF token to enable SR-IOV
> >support since 5.7.
> >
> >The VF token can be set by a vfio-pci based PF driver and must be known
> >by the vfio-pci based VF driver in order to gain access to the device.
> >
> >Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> >Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> >Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> >---
> 
> [snip]
> 
> >+
> >+/**
> >+ * Copy the user-configured vfio VF token.
> >+ *
> >+ * @param vf_token
> >+ *   vfio VF token configured with the command line is copied
> >+ *   into this parameter, zero uuid by default.
> >+ */
> >+__rte_experimental
> >+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
> >+
> 
> Minor nit, missing the EXPERIMENTAL warning int the comment.
> 

Yes, missed, it is good for kindly reminding. ;-)

>  * @warning
>  * @b EXPERIMENTAL: this API may change without prior notice
>  *

[dpdk-dev] [PATCH v15 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v15: Add the missed EXPERIMENTAL warning for API doxgen.

v14: Rebase the patch for 20.08 release note.

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
     device needs to be specified with this token. Also no ABI issue
     now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.


Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 119 insertions(+), 1 deletion(-)

-- 
2.26.2

[dpdk-dev] [PATCH v15 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.26.2

[dpdk-dev] [PATCH v15 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 117 insertions(+), 1 deletion(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index d40b495c1..02fba7d1e 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/linux_gsg/linux_eal_parameters.rst b/doc/guides/linux_gsg/linux_eal_parameters.rst
index b2cc60e44..bd3977cb3 100644
--- a/doc/guides/linux_gsg/linux_eal_parameters.rst
+++ b/doc/guides/linux_gsg/linux_eal_parameters.rst
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/rel_notes/release_20_08.rst b/doc/guides/rel_notes/release_20_08.rst
index 39064afbe..dfce08b89 100644
--- a/doc/guides/rel_notes/release_20_08.rst
+++ b/doc/guides/rel_notes/release_20_08.rst
@@ -56,6 +56,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
 
 Removed Items
 -------------
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 8f2cbd1c6..e7b643a99 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -92,6 +92,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -297,6 +298,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index c650bc081..43095b7a2 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 18e6da9ab..89769d48b 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index c41f265fa..4ff5028d8 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
index 2f9ed298d..2f47347dc 100644
--- a/lib/librte_eal/include/rte_eal.h
+++ b/lib/librte_eal/include/rte_eal.h
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index f162124a3..00f3fdced 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index d26e1649a..5b2f6b305 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index d8038749a..93a4d30f8 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -372,6 +372,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
-- 
2.26.2

[dpdk-dev] [PATCH v16 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v16: Rebase the patch for 20.08 release note.

v15: Add the missed EXPERIMENTAL warning for API doxgen.

v14: Rebase the patch for 20.08 release note.

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
     device needs to be specified with this token. Also no ABI issue
     now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 119 insertions(+), 1 deletion(-)

-- 
2.27.0

[dpdk-dev] [PATCH v16 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.27.0

[dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 117 insertions(+), 1 deletion(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index d40b495c1..02fba7d1e 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/linux_gsg/linux_eal_parameters.rst b/doc/guides/linux_gsg/linux_eal_parameters.rst
index b2cc60e44..bd3977cb3 100644
--- a/doc/guides/linux_gsg/linux_eal_parameters.rst
+++ b/doc/guides/linux_gsg/linux_eal_parameters.rst
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/rel_notes/release_20_08.rst b/doc/guides/rel_notes/release_20_08.rst
index 86d240213..ca7fd521b 100644
--- a/doc/guides/rel_notes/release_20_08.rst
+++ b/doc/guides/rel_notes/release_20_08.rst
@@ -62,6 +62,11 @@ New Features
 
   * Added new PMD devarg ``reclaim_mem_mode``.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
 
 Removed Items
 -------------
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 0546beb3a..fc889309c 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -94,6 +94,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -301,6 +302,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index c650bc081..43095b7a2 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 18e6da9ab..89769d48b 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index c41f265fa..4ff5028d8 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
index 2f9ed298d..2f47347dc 100644
--- a/lib/librte_eal/include/rte_eal.h
+++ b/lib/librte_eal/include/rte_eal.h
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index f162124a3..00f3fdced 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index d26e1649a..5b2f6b305 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 196eef5af..16c8de27e 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -372,6 +372,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
-- 
2.27.0

Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token

[Harman Kalra]

On Wed, Jun 17, 2020 at 02:33:21PM +0800, Haiyue Wang wrote:
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support since 5.7.
> 
> The VF token can be set by a vfio-pci based PF driver and must be known
> by the vfio-pci based VF driver in order to gain access to the device.
> 
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> ---

Tested-by: Harman Kalra <hkalra@marvell.com>

>  doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
>  doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
>  doc/guides/rel_notes/release_20_08.rst        |  5 +++
>  lib/librte_eal/common/eal_common_options.c    |  2 ++
>  lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
>  lib/librte_eal/common/eal_options.h           |  2 ++
>  lib/librte_eal/freebsd/eal.c                  |  4 +++
>  lib/librte_eal/include/rte_eal.h              | 15 ++++++++
>  lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
>  lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
>  lib/librte_eal/rte_eal_version.map            |  1 +
>  11 files changed, 117 insertions(+), 1 deletion(-)
> 
> diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
> index d40b495c1..02fba7d1e 100644
> --- a/doc/guides/linux_gsg/linux_drivers.rst
> +++ b/doc/guides/linux_gsg/linux_drivers.rst
> @@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
>  VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
>  however please consult your distributions documentation to make sure that is the case.
>  
> +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
> +functions. After the PF is bound to vfio-pci module, the user can create the VFs
> +by sysfs interface, and these VFs are bound to vfio-pci module automatically.
> +
> +When the PF is bound to vfio-pci, it has initial VF token generated by random. For
> +security reason, this token is write only, the user can't read it from the kernel
> +directly. For accessing the VF, the user needs to start the PF with token parameter
> +to setup a VF token (uuid format), then the VF can be accessed with this new known
> +VF token.
> +
> +DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
> +PF and its related VFs, this VF token will be shared in all VFIO devices, including
> +the different PFs.
> +
> +.. code-block:: console
> +
> +    1. Generate the VF token by uuid command
> +        14d63f20-8445-11ea-8900-1f9ce7d5650d
> +
> +    2. sudo modprobe vfio-pci enable_sriov=1
> +
> +    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
> +
> +    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
> +
> +    4. Start the PF:
> +        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
> +         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
> +
> +    5. Start the VF:
> +        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
> +         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
> +
>  Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
>  
>  .. note::
>  
> -    ``vfio-pci`` module doesn't support the creation of virtual functions.
> +    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
>  
>  For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
>  This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
> diff --git a/doc/guides/linux_gsg/linux_eal_parameters.rst b/doc/guides/linux_gsg/linux_eal_parameters.rst
> index b2cc60e44..bd3977cb3 100644
> --- a/doc/guides/linux_gsg/linux_eal_parameters.rst
> +++ b/doc/guides/linux_gsg/linux_eal_parameters.rst
> @@ -40,6 +40,10 @@ Device-related options
>  
>      Use specified interrupt mode for devices bound to VFIO kernel driver.
>  
> +*   ``--vfio-vf-token <uuid>``
> +
> +    Use specified VF token for devices bound to VFIO kernel driver.
> +
>  Multiprocessing-related options
>  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>  
> diff --git a/doc/guides/rel_notes/release_20_08.rst b/doc/guides/rel_notes/release_20_08.rst
> index 86d240213..ca7fd521b 100644
> --- a/doc/guides/rel_notes/release_20_08.rst
> +++ b/doc/guides/rel_notes/release_20_08.rst
> @@ -62,6 +62,11 @@ New Features
>  
>    * Added new PMD devarg ``reclaim_mem_mode``.
>  
> +* **Added the support for vfio-pci new VF token interface.**
> +
> +  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
> +  the trust between SR-IOV PF and the created VFs. Update the method to gain access
> +  to the PF and VFs devices by appending the VF token parameter.
>  
>  Removed Items
>  -------------
> diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
> index 0546beb3a..fc889309c 100644
> --- a/lib/librte_eal/common/eal_common_options.c
> +++ b/lib/librte_eal/common/eal_common_options.c
> @@ -94,6 +94,7 @@ eal_long_options[] = {
>  	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
>  	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
>  	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
> +	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
>  	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
>  	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
>  	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
> @@ -301,6 +302,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
>  
>  	/* if set to NONE, interrupt mode is determined automatically */
>  	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
> +	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
>  
>  #ifdef RTE_LIBEAL_USE_HPET
>  	internal_cfg->no_hpet = 0;
> diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
> index c650bc081..43095b7a2 100644
> --- a/lib/librte_eal/common/eal_internal_cfg.h
> +++ b/lib/librte_eal/common/eal_internal_cfg.h
> @@ -72,6 +72,8 @@ struct internal_config {
>  	volatile int syslog_facility;	  /**< facility passed to openlog() */
>  	/** default interrupt mode for VFIO */
>  	volatile enum rte_intr_mode vfio_intr_mode;
> +	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
> +	rte_uuid_t vfio_vf_token;
>  	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
>  	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
>  	char *user_mbuf_pool_ops_name;
> diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
> index 18e6da9ab..89769d48b 100644
> --- a/lib/librte_eal/common/eal_options.h
> +++ b/lib/librte_eal/common/eal_options.h
> @@ -69,6 +69,8 @@ enum {
>  	OPT_VDEV_NUM,
>  #define OPT_VFIO_INTR         "vfio-intr"
>  	OPT_VFIO_INTR_NUM,
> +#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
> +	OPT_VFIO_VF_TOKEN_NUM,
>  #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
>  	OPT_VMWARE_TSC_MAP_NUM,
>  #define OPT_LEGACY_MEM    "legacy-mem"
> diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
> index c41f265fa..4ff5028d8 100644
> --- a/lib/librte_eal/freebsd/eal.c
> +++ b/lib/librte_eal/freebsd/eal.c
> @@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)
>  	return RTE_INTR_MODE_NONE;
>  }
>  
> +void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
> +{
> +}
> +
>  int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
>  		      __rte_unused const char *dev_addr,
>  		      __rte_unused int *vfio_dev_fd,
> diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
> index 2f9ed298d..2f47347dc 100644
> --- a/lib/librte_eal/include/rte_eal.h
> +++ b/lib/librte_eal/include/rte_eal.h
> @@ -21,6 +21,7 @@
>  #include <rte_bus.h>
>  
>  #include <rte_pci_dev_feature_defs.h>
> +#include <rte_uuid.h>
>  
>  #ifdef __cplusplus
>  extern "C" {
> @@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);
>   */
>  enum rte_intr_mode rte_eal_vfio_intr_mode(void);
>  
> +
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Copy the user-configured vfio VF token.
> + *
> + * @param vf_token
> + *   vfio VF token configured with the command line is copied
> + *   into this parameter, zero uuid by default.
> + */
> +__rte_experimental
> +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
> +
>  /**
>   * A wrap API for syscall gettid.
>   *
> diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
> index f162124a3..00f3fdced 100644
> --- a/lib/librte_eal/linux/eal.c
> +++ b/lib/librte_eal/linux/eal.c
> @@ -558,6 +558,7 @@ eal_usage(const char *prgname)
>  	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
>  	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
>  	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
> +	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
>  	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
>  	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
>  	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
> @@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
>  	return -1;
>  }
>  
> +static int
> +eal_parse_vfio_vf_token(const char *vf_token)
> +{
> +	rte_uuid_t uuid;
> +
> +	if (!rte_uuid_parse(vf_token, uuid)) {
> +		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
> +		return 0;
> +	}
> +
> +	return -1;
> +}
> +
>  /* Parse the arguments for --log-level only */
>  static void
>  eal_log_level_parse(int argc, char **argv)
> @@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv)
>  			}
>  			break;
>  
> +		case OPT_VFIO_VF_TOKEN_NUM:
> +			if (eal_parse_vfio_vf_token(optarg) < 0) {
> +				RTE_LOG(ERR, EAL, "invalid parameters for --"
> +						OPT_VFIO_VF_TOKEN "\n");
> +				eal_usage(prgname);
> +				ret = -1;
> +				goto out;
> +			}
> +			break;
> +
>  		case OPT_CREATE_UIO_DEV_NUM:
>  			internal_config.create_uio_dev = 1;
>  			break;
> @@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)
>  	return internal_config.vfio_intr_mode;
>  }
>  
> +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
> +{
> +	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
> +}
> +
>  int
>  rte_eal_check_module(const char *module_name)
>  {
> diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
> index d26e1649a..5b2f6b305 100644
> --- a/lib/librte_eal/linux/eal_vfio.c
> +++ b/lib/librte_eal/linux/eal_vfio.c
> @@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  	int vfio_container_fd;
>  	int vfio_group_fd;
>  	int iommu_group_num;
> +	rte_uuid_t vf_token;
>  	int i, ret;
>  
>  	/* get group number */
> @@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  				t->type_id, t->name);
>  	}
>  
> +	rte_eal_vfio_get_vf_token(vf_token);
> +
> +	/* get a file descriptor for the device with VF token firstly */
> +	if (!rte_uuid_is_null(vf_token)) {
> +		char vf_token_str[RTE_UUID_STRLEN];
> +		char dev[PATH_MAX];
> +
> +		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
> +		snprintf(dev, sizeof(dev),
> +			 "%s vf_token=%s", dev_addr, vf_token_str);
> +
> +		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
> +				     dev);
> +		if (*vfio_dev_fd >= 0)
> +			goto dev_get_info;
> +	}
> +
>  	/* get a file descriptor for the device */
>  	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
>  	if (*vfio_dev_fd < 0) {
> @@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
>  		return -1;
>  	}
>  
> +dev_get_info:
>  	/* test and setup the device */
>  	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
>  	if (ret) {
> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index 196eef5af..16c8de27e 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -372,6 +372,7 @@ EXPERIMENTAL {
>  	__rte_trace_point_register;
>  	per_lcore_trace_mem;
>  	per_lcore_trace_point_sz;
> +	rte_eal_vfio_get_vf_token;
>  	rte_log_can_log;
>  	rte_thread_getname;
>  	rte_trace_dump;
> -- 
> 2.27.0
> 

Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token

[David Marchand]

On Mon, Jun 22, 2020 at 10:40 PM Harman Kalra <hkalra@marvell.com> wrote:
>
> On Wed, Jun 17, 2020 at 02:33:21PM +0800, Haiyue Wang wrote:
> > The kernel module vfio-pci introduces the VF token to enable SR-IOV
> > support since 5.7.
> >
> > The VF token can be set by a vfio-pci based PF driver and must be known
> > by the vfio-pci based VF driver in order to gain access to the device.
> >
> > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> > Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> > ---
>
> Tested-by: Harman Kalra <hkalra@marvell.com>

Thanks for the test Harman.

I can see no complaint on using a single token for all devices, which
is the only concern I would have with the last revision.
If everyone is ok with this choice, I will take this for -rc1.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token

[Wang, Haiyue]

> -----Original Message-----
> From: David Marchand <david.marchand@redhat.com>
> Sent: Thursday, June 25, 2020 15:33
> To: Harman Kalra <hkalra@marvell.com>; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
> Cc: Wang, Haiyue <haiyue.wang@intel.com>; dev <dev@dpdk.org>; Burakov, Anatoly
> <anatoly.burakov@intel.com>; Thomas Monjalon <thomas@monjalon.net>; Andrew Rybchenko
> <arybchenko@solarflare.com>; Maxime Coquelin <maxime.coquelin@redhat.com>
> Subject: Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token
> 
> On Mon, Jun 22, 2020 at 10:40 PM Harman Kalra <hkalra@marvell.com> wrote:
> >
> > On Wed, Jun 17, 2020 at 02:33:21PM +0800, Haiyue Wang wrote:
> > > The kernel module vfio-pci introduces the VF token to enable SR-IOV
> > > support since 5.7.
> > >
> > > The VF token can be set by a vfio-pci based PF driver and must be known
> > > by the vfio-pci based VF driver in order to gain access to the device.
> > >
> > > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > > Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> > > Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> > > ---
> >
> > Tested-by: Harman Kalra <hkalra@marvell.com>
> 
> Thanks for the test Harman.
> 
> I can see no complaint on using a single token for all devices, which

Yeah, not the best, but may meet with the most needed: create VFs by vfio-pci ;-)

Since devarg was a private option for ALL kind of devices, and have to break the
ABI policy to implement the design, so I drop this revision, choose to use it as
a global option as "vfio-intr" option does.

> is the only concern I would have with the last revision.
> If everyone is ok with this choice, I will take this for -rc1.
> 
> 
> --
> David Marchand

[dpdk-dev] [PATCH v17 0/2] support for VFIO-PCI VF token interface

[Haiyue Wang]

v17: Rebase for new EAL config API, update the commit message and doc.

v16: Rebase the patch for 20.08 release note.

v15: Add the missed EXPERIMENTAL warning for API doxgen.

v14: Rebase the patch for 20.08 release note.

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
     device needs to be specified with this token. Also no ABI issue
     now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
    document.

v6: Drop the Fixes tag in uuid, since the file has been
    moved to another place, not suitable to apply on stable.
    And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
    2. Split into two patches for different logic module.
    3. Add more comments into the code for explaining the design.
    4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
       for Linux driver use.

v3: Fix the Travis build failed:
           (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
           (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
         Based on Vamsi's RFC v1, and Alex's patch for Qemu
        [https://lore.kernel.org/lkml/20200204161737.34696b91@w520.home/]: 
       Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
  eal: add uuid dependent header files explicitly
  eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  3 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 14 ++++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 33 +++++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  3 ++
 12 files changed, 127 insertions(+), 1 deletion(-)

-- 
2.27.0

[dpdk-dev] [PATCH v17 1/2] eal: add uuid dependent header files explicitly

[Haiyue Wang]

Add the dependent header files explicitly, so that the user just needs
to include the 'rte_uuid.h' header file directly to avoid compile error:
 (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
 (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes: 6bc67c497a51 ("eal: add uuid API")
Cc: stable@dpdk.org

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
index 044afbdfa..8b42e070a 100644
--- a/lib/librte_eal/include/rte_uuid.h
+++ b/lib/librte_eal/include/rte_uuid.h
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
-- 
2.27.0

Re: [dpdk-dev] [PATCH v17 1/2] eal: add uuid dependent header files explicitly

[David Marchand]

On Fri, Jul 3, 2020 at 5:05 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
>
> Add the dependent header files explicitly, so that the user just needs
> to include the 'rte_uuid.h' header file directly to avoid compile error:
>  (1). rte_uuid.h:97:55: error: unknown type name ‘size_t’
>  (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
>
> Fixes: 6bc67c497a51 ("eal: add uuid API")
> Cc: stable@dpdk.org
>
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> ---
>  lib/librte_eal/include/rte_uuid.h | 2 ++
>  1 file changed, 2 insertions(+)
>
> diff --git a/lib/librte_eal/include/rte_uuid.h b/lib/librte_eal/include/rte_uuid.h
> index 044afbdfa..8b42e070a 100644
> --- a/lib/librte_eal/include/rte_uuid.h
> +++ b/lib/librte_eal/include/rte_uuid.h
> @@ -15,6 +15,8 @@ extern "C" {
>  #endif
>
>  #include <stdbool.h>
> +#include <stddef.h>
> +#include <string.h>
>
>  /**
>   * Struct describing a Universal Unique Identifier
> --
> 2.27.0
>

Acked-by: David Marchand <david.marchand@redhat.com>


-- 
David Marchand

[dpdk-dev] [PATCH v17 2/2] eal: support for VFIO-PCI VF token

[Haiyue Wang]

The kernel module vfio-pci introduces the VF token to enable SR-IOV
support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known
by the vfio-pci based VF driver in order to gain access to the device.

Since the vfio-pci module uses the VF token as internal data to provide
the collaboration between SR-IOV PF and VFs, so DPDK can use the same
VF token for all PF devices by specifying the related EAL option.

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
Tested-by: Harman Kalra <hkalra@marvell.com>
---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  3 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 14 ++++++++
 lib/librte_eal/linux/eal.c                    | 33 +++++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  3 ++
 11 files changed, 125 insertions(+), 1 deletion(-)

diff --git a/doc/guides/linux_gsg/linux_drivers.rst b/doc/guides/linux_gsg/linux_drivers.rst
index d40b495c1..4eda3d5bf 100644
--- a/doc/guides/linux_gsg/linux_drivers.rst
+++ b/doc/guides/linux_gsg/linux_drivers.rst
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. To access the VF, the user needs to start the PF with token parameter to
+setup a VF token in UUID format, then the VF can be accessed with this new token.
+
+Since the ``vfio-pci`` module uses the VF token as internal data to provide the
+collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all
+PF devices which bound to one application. This VF token can be specified by the EAL
+parameter ``--vfio-vf-token``.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
diff --git a/doc/guides/linux_gsg/linux_eal_parameters.rst b/doc/guides/linux_gsg/linux_eal_parameters.rst
index b2cc60e44..bd3977cb3 100644
--- a/doc/guides/linux_gsg/linux_eal_parameters.rst
+++ b/doc/guides/linux_gsg/linux_eal_parameters.rst
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/guides/rel_notes/release_20_08.rst b/doc/guides/rel_notes/release_20_08.rst
index 5cbc4ce14..333f315ca 100644
--- a/doc/guides/rel_notes/release_20_08.rst
+++ b/doc/guides/rel_notes/release_20_08.rst
@@ -107,6 +107,12 @@ New Features
   * Dump ``rte_flow`` memory consumption.
   * Measure packet per second forwarding.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  From Linux 5.7, vfio-pci supports to bind both SR-IOV PF and the created VFs,
+  it uses a shared VF token (UUID) to represent the collaboration between PF
+  and VFs. Update DPDK PCI driver to gain the access to the PF and VFs devices
+  by appending the VF token parameter.
 
 Removed Items
 -------------
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 24b223ebf..15b47349d 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -94,6 +94,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -327,6 +328,8 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0,
+			sizeof(internal_cfg->vfio_vf_token));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index 2aaa167d4..13f93388a 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
index 18e6da9ab..89769d48b 100644
--- a/lib/librte_eal/common/eal_options.h
+++ b/lib/librte_eal/common/eal_options.h
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index 8c75cba79..040b1b7ee 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -966,6 +966,11 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void
+rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
diff --git a/lib/librte_eal/include/rte_eal.h b/lib/librte_eal/include/rte_eal.h
index 2f9ed298d..6c70632cc 100644
--- a/lib/librte_eal/include/rte_eal.h
+++ b/lib/librte_eal/include/rte_eal.h
@@ -19,6 +19,7 @@
 #include <rte_compat.h>
 #include <rte_per_lcore.h>
 #include <rte_bus.h>
+#include <rte_uuid.h>
 
 #include <rte_pci_dev_feature_defs.h>
 
@@ -438,6 +439,19 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index 3b56d14da..d6b8d8be7 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -540,6 +540,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -620,6 +621,20 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	struct internal_config *cfg = eal_get_internal_configuration();
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(cfg->vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -762,6 +777,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_conf->create_uio_dev = 1;
 			break;
@@ -1342,6 +1367,14 @@ rte_eal_vfio_intr_mode(void)
 	return internal_conf->vfio_intr_mode;
 }
 
+void
+rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	struct internal_config *cfg = eal_get_internal_configuration();
+
+	rte_uuid_copy(vf_token, cfg->vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
diff --git a/lib/librte_eal/linux/eal_vfio.c b/lib/librte_eal/linux/eal_vfio.c
index abb12a354..d4470a34d 100644
--- a/lib/librte_eal/linux/eal_vfio.c
+++ b/lib/librte_eal/linux/eal_vfio.c
@@ -714,6 +714,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 	const struct internal_config *internal_conf =
 		eal_get_internal_configuration();
@@ -899,6 +900,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -914,6 +932,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	}
 
 	/* test and setup the device */
+dev_get_info:
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
 		RTE_LOG(ERR, EAL, "  %s cannot get device info, "
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 196eef5af..7923c62f2 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -386,6 +386,9 @@ EXPERIMENTAL {
 	rte_trace_point_lookup;
 	rte_trace_regexp;
 	rte_trace_save;
+
+	# added in 20.08
+	rte_eal_vfio_get_vf_token;
 };
 
 INTERNAL {
-- 
2.27.0

Re: [dpdk-dev] [PATCH v17 2/2] eal: support for VFIO-PCI VF token

[David Marchand]

On Fri, Jul 3, 2020 at 5:05 PM Haiyue Wang <haiyue.wang@intel.com> wrote:
>
> The kernel module vfio-pci introduces the VF token to enable SR-IOV
> support since 5.7.
>
> The VF token can be set by a vfio-pci based PF driver and must be known
> by the vfio-pci based VF driver in order to gain access to the device.
>
> Since the vfio-pci module uses the VF token as internal data to provide
> the collaboration between SR-IOV PF and VFs, so DPDK can use the same
> VF token for all PF devices by specifying the related EAL option.
>
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
> Acked-by: Andrew Rybchenko <arybchenko@solarflare.com>
> Tested-by: Harman Kalra <hkalra@marvell.com>

Series applied, thanks Haiyue.


-- 
David Marchand