[dpdk-dev] [PATCH 20.11 11/20] raw/ioat: create rawdev instances for idxd vdevs

[Bruce Richardson <bruce.richardson@intel.com>]

From: Kevin Laatz <kevin.laatz@intel.com>

For each vdev (DSA work queue) instance, create a rawdev instance. Since
the vdev support depends upon the accel-config libraries, make the vdev
support conditional upon that in meson.build.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
 drivers/raw/ioat/idxd_vdev.c | 163 ++++++++++++++++++++++++++++++++++-
 drivers/raw/ioat/meson.build |   9 +-
 2 files changed, 169 insertions(+), 3 deletions(-)

diff --git a/drivers/raw/ioat/idxd_vdev.c b/drivers/raw/ioat/idxd_vdev.c
index 73fce6d87..e81bd7326 100644
--- a/drivers/raw/ioat/idxd_vdev.c
+++ b/drivers/raw/ioat/idxd_vdev.c
@@ -2,6 +2,12 @@
  * Copyright(c) 2020 Intel Corporation
  */
 
+#include <fcntl.h>
+#include <limits.h>
+#include <sys/mman.h>
+#include <accel-config/libaccel_config.h>
+
+#include <rte_memzone.h>
 #include <rte_bus_vdev.h>
 #include <rte_kvargs.h>
 #include <rte_string_fns.h>
@@ -26,6 +32,79 @@ struct idxd_vdev_args {
 	uint8_t wq_id;
 };
 
+static const struct rte_rawdev_ops idxd_vdev_ops = {
+		.dev_selftest = idxd_rawdev_test,
+};
+
+static void *
+idxd_vdev_mmap_wq(struct accfg_device *dsa_dev, struct accfg_wq *wq)
+{
+	void *addr;
+	int major, minor;
+	char path[PATH_MAX];
+	int fd;
+
+	major = accfg_device_get_cdev_major(dsa_dev);
+	if (major < 0) {
+		IOAT_PMD_ERR("Invalid major version %d", major);
+		return NULL;
+	}
+
+	minor = accfg_wq_get_cdev_minor(wq);
+	if (minor < 0) {
+		IOAT_PMD_ERR("Invalid minor version %d", minor);
+		return NULL;
+	}
+
+	snprintf(path, sizeof(path), "/dev/char/%u:%u", major, minor);
+	fd = open(path, O_RDWR);
+	if (fd < 0) {
+		IOAT_PMD_ERR("Failed to open device path");
+		return NULL;
+	}
+
+	addr = mmap(NULL, 0x1000, PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd, 0);
+	if (addr == MAP_FAILED) {
+		IOAT_PMD_ERR("Failed to mmap device");
+		return NULL;
+	}
+
+	return addr;
+}
+
+static int
+idxd_rawdev_vdev_config(struct idxd_rawdev *idxd, struct idxd_vdev_args *args)
+{
+	struct accfg_ctx *dsa_ctx;
+	struct accfg_device *dsa_dev;
+	struct accfg_wq *dsa_wq;
+	int ret;
+
+	ret = accfg_new(&dsa_ctx);
+	if (ret < 0) {
+		IOAT_PMD_ERR("Failed to create device context");
+		ret = -ENOMEM;
+	}
+
+	dsa_dev = accfg_ctx_device_get_by_id(dsa_ctx, args->device_id);
+	if (dsa_dev == NULL) {
+		IOAT_PMD_ERR("device not found: %u", args->device_id);
+		return -1;
+	}
+	dsa_wq = accfg_device_wq_get_by_id(dsa_dev, args->wq_id);
+	if (dsa_wq == NULL) {
+		IOAT_PMD_ERR("queue not found: %u", args->wq_id);
+		return -1;
+	}
+
+	idxd->u.vdev.ctx = dsa_ctx;
+	idxd->u.vdev.device = dsa_dev;
+	idxd->u.vdev.wq = dsa_wq;
+	idxd->max_batches = accfg_wq_get_size(dsa_wq);
+
+	return ret;
+}
+
 static int
 idxd_rawdev_parse_wq(const char *key __rte_unused, const char *value,
 			  void *extra_args)
@@ -76,6 +155,7 @@ static int
 idxd_rawdev_probe_vdev(struct rte_vdev_device *vdev)
 {
 	struct rte_kvargs *kvlist;
+	struct idxd_rawdev idxd = {0};
 	struct idxd_vdev_args vdev_args;
 	const char *name;
 	int ret = 0;
@@ -98,15 +178,52 @@ idxd_rawdev_probe_vdev(struct rte_vdev_device *vdev)
 		return -EINVAL;
 	}
 
+	ret = idxd_rawdev_vdev_config(&idxd, &vdev_args);
+	if (ret) {
+		IOAT_PMD_ERR("Failed to init vdev context");
+		return ret;
+	}
+
+	idxd.qid = vdev_args.wq_id;
+	idxd.public.portal = idxd_vdev_mmap_wq(idxd.u.vdev.device, idxd.u.vdev.wq);
+	if (idxd.public.portal == NULL) {
+		IOAT_PMD_ERR("WQ mmap failed");
+		return -ENOMEM;
+	}
+
 	vdev->device.driver = &idxd_rawdev_drv_vdev.driver;
 
+	ret = idxd_rawdev_create(name, &vdev->device, &idxd, &idxd_vdev_ops);
+	if (ret) {
+		IOAT_PMD_ERR("Failed to create rawdev %s", name);
+		return ret;
+	}
+
+	/* enable the device itself */
+	if (accfg_device_is_active(idxd.u.vdev.device)) {
+		IOAT_PMD_INFO("Device %s already enabled",
+				accfg_device_get_devname(idxd.u.vdev.device));
+	} else {
+		ret = accfg_device_enable(idxd.u.vdev.device);
+		if (ret) {
+			IOAT_PMD_ERR("Error enabling device %s",
+					accfg_device_get_devname(idxd.u.vdev.device));
+			return -1;
+		}
+		IOAT_PMD_DEBUG("Enabling device %s OK",
+				accfg_device_get_devname(idxd.u.vdev.device));
+	}
+
 	return 0;
 }
 
 static int
 idxd_rawdev_remove_vdev(struct rte_vdev_device *vdev)
 {
+	struct idxd_rawdev *idxd;
 	const char *name;
+	struct rte_rawdev *rdev;
+	int ret = 0;
 
 	name = rte_vdev_device_name(vdev);
 	if (name == NULL)
@@ -114,7 +231,51 @@ idxd_rawdev_remove_vdev(struct rte_vdev_device *vdev)
 
 	IOAT_PMD_INFO("Remove DSA vdev %p", name);
 
-	return 0;
+	rdev = rte_rawdev_pmd_get_named_dev(name);
+	if (!rdev) {
+		IOAT_PMD_ERR("Invalid device name (%s)", name);
+		return -EINVAL;
+	}
+
+	idxd = rdev->dev_private;
+
+	/* disable the device */
+	if (!accfg_device_is_active(idxd->u.vdev.device)) {
+		IOAT_PMD_ERR("Device %s already disabled",
+				accfg_device_get_devname(idxd->u.vdev.device));
+	}
+
+	ret = accfg_device_disable(idxd->u.vdev.device);
+	if (ret) {
+		IOAT_PMD_ERR("Not able to disable device %s",
+				accfg_device_get_devname(idxd->u.vdev.device));
+		return ret;
+	}
+	IOAT_PMD_DEBUG("Disabling device %s OK",
+			accfg_device_get_devname(idxd->u.vdev.device));
+
+	/* free context and memory */
+	if (rdev->dev_private != NULL) {
+		IOAT_PMD_DEBUG("Freeing device driver memory");
+		rdev->dev_private = NULL;
+		accfg_unref(idxd->u.vdev.ctx);
+
+		if (munmap(idxd->public.portal, 0x1000) < 0) {
+			IOAT_PMD_ERR("Error unmapping %s",
+				accfg_wq_get_devname(idxd->u.vdev.wq));
+			ret = -errno;
+		}
+
+		rte_free(idxd->public.batch_ring);
+		rte_free(idxd->public.hdl_ring);
+
+		rte_memzone_free(idxd->mz);
+	}
+
+	if (rte_rawdev_pmd_release(rdev))
+		IOAT_PMD_ERR("Device cleanup failed");
+
+	return ret;
 }
 
 struct rte_vdev_driver idxd_rawdev_drv_vdev = {
diff --git a/drivers/raw/ioat/meson.build b/drivers/raw/ioat/meson.build
index 5eff76a1a..a730953f8 100644
--- a/drivers/raw/ioat/meson.build
+++ b/drivers/raw/ioat/meson.build
@@ -5,14 +5,19 @@ build = dpdk_conf.has('RTE_ARCH_X86')
 reason = 'only supported on x86'
 sources = files(
 	'idxd_pci.c',
-	'idxd_vdev.c',
 	'ioat_common.c',
 	'ioat_rawdev.c',
 	'ioat_rawdev_test.c')
 deps += ['bus_pci',
-	'bus_vdev',
 	'mbuf',
 	'rawdev']
 
 install_headers('rte_ioat_rawdev.h',
 		'rte_ioat_rawdev_fns.h')
+
+accfg_dep = dependency('libaccel-config', required: false)
+if accfg_dep.found()
+	sources += files('idxd_vdev.c')
+	deps += ['bus_vdev']
+	ext_deps += accfg_dep
+endif
-- 
2.25.1