[dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

[<vattunuru@marvell.com>]

From: Vamsi Attunuru <vattunuru@marvell.com>

vfio-pci driver enables virtual function access from
the DPDK applications when those vf device's physical
function is also bound to vfio driver.

Patch adds the required configuration and checks to
enable DPDK applications to access both pf and it's
vf devices through vfio-pci driver.

See background on vf token scheme in linux vfio driver.
http://patches.dpdk.org/cover/65915/

When a physical function is enabled with  non-zero
virtual functions,  patch sets the UUID using
VFIO_DEVICE_FEATURE ioctl from physical function's
file descriptor. Same UUID is used to gain the access
for the virtual functions on those  physical function.

Following changes required on top of this DPDK patch
* Kernel version check for VFIO_DEVICE_FEATURE ioctl
* Use uuid gen API to generate UUID.

Signed-off-by: Vamsi Attunuru <vattunuru@marvell.com>

diff --git a/lib/librte_eal/linux/eal/eal_vfio.c b/lib/librte_eal/linux/eal/eal_vfio.c
index 01b5ef3..e2fdd35 100644
--- a/lib/librte_eal/linux/eal/eal_vfio.c
+++ b/lib/librte_eal/linux/eal/eal_vfio.c
@@ -12,6 +12,7 @@
 #include <rte_log.h>
 #include <rte_memory.h>
 #include <rte_eal_memconfig.h>
+#include <rte_uuid.h>
 #include <rte_vfio.h>
 
 #include "eal_filesystem.h"
@@ -50,6 +51,9 @@ struct vfio_config {
 	struct user_mem_maps mem_maps;
 };
 
+rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
+				      0x913f, 0xf2d2f965ed0eULL);
+
 /* per-process VFIO config */
 static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];
 static struct vfio_config *default_vfio_cfg = &vfio_cfgs[0];
@@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)
 	return 0;
 }
 
+static bool
+rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	/* check if physfn directory exist for this device */
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+
+	/* For PFs, physfn directory does not exist */
+	if (ret < 0)
+		return true;
+
+	return false;
+}
+
+static int
+is_vf_token_required(const char *sysfs_base, const char *dev_addr)
+{
+	char *tok[16], *physfn, *physfn_drv;
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
+		return -1;
+	}
+
+	physfn = tok[ret - 1];
+
+	snprintf(linkname, sizeof(linkname),
+			 "/sys/bus/pci/devices/%s/driver", physfn);
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
+			dev_addr);
+		return -1;
+	}
+
+	physfn_drv = tok[ret - 1];
+
+	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
+		return 1;
+
+	/* physfn is bound to vfio-pci */
+	return 0;
+}
+
+static bool
+rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	unsigned long num_vfs;
+	int ret;
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
+		return false;
+
+	memset(linkname, 0, sizeof(linkname));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
+
+	ret  = eal_parse_sysfs_value(linkname, &num_vfs);
+
+	if ((ret < 0) || (num_vfs == 0))
+		return false;
+
+	return true;
+}
+
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		int *vfio_dev_fd, struct vfio_device_info *device_info)
@@ -669,6 +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	if (ret < 0)
 		return -1;
 
+	snprintf(dev, sizeof(dev), "%s", dev_addr);
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
+		char vf_token[PATH_MAX];
+		/*
+		 *  Check if vf_token is required or not,
+		 *  vf_token is required when the VF's physfn is
+		 *  binded with vfio-pci driver
+		 */
+		ret = is_vf_token_required(sysfs_base, dev_addr);
+		/* if negative, something failed */
+		if (ret < 0)
+			return -1;
+
+		if (ret == 0) {
+			/* vf_token required to open device file descriptor */
+			rte_uuid_unparse(uuid_token,
+					 vf_token, sizeof(vf_token));
+			snprintf(dev, sizeof(dev),
+				 "%s vf_token=%s", dev_addr, vf_token);
+		}
+	}
+
 	/* get the actual group fd */
 	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
 	if (vfio_group_fd < 0)
@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	}
 
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		rte_vfio_clear_group(vfio_group_fd);
 		return -1;
 	}
+
+	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {
+#define VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
+
+		struct vfio_device_feature *vf_token;
+		uint8_t local[VF_TOKEN];
+
+		memset(local, 0, VF_TOKEN);
+		vf_token = (struct vfio_device_feature *)local;
+		vf_token->argsz = VF_TOKEN;
+		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
+					VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
+
+		memcpy(local + sizeof(struct vfio_device_feature),
+		       &uuid_token, sizeof(uuid_token));
+
+		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
+		if (ret) {
+			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
+					"error %i (%s)\n", dev_addr, errno,
+					strerror(errno));
+			return -1;
+		}
+	}
+
 	vfio_group_device_get(vfio_group_fd);
 
 	return 0;
-- 
2.8.4