DPDK patches and discussions
 help / color / mirror / Atom feed
* [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus
@ 2021-03-11 13:01 Xueming Li
  2021-04-12  8:29 ` Xueming(Steven) Li
  2021-04-13  3:23 ` [dpdk-dev] [PATCH v1] " Xueming Li
  0 siblings, 2 replies; 28+ messages in thread
From: Xueming Li @ 2021-03-11 13:01 UTC (permalink / raw)
  To: NBU-Contact-Thomas Monjalon
  Cc: dev, xuemingl, Parav Pandit, Ray Kinsella, Neil Horman

Auxiliary [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary_device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 MAINTAINERS                               |   5 +
 drivers/bus/auxiliary/auxiliary_common.c  | 410 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 144 ++++++++
 drivers/bus/auxiliary/meson.build         |  17 +
 drivers/bus/auxiliary/private.h           | 119 +++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 193 ++++++++++
 drivers/bus/auxiliary/version.map         |  11 +
 drivers/bus/meson.build                   |   2 +-
 9 files changed, 958 insertions(+), 1 deletion(-)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index e341bc81d5..00db01e471 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -514,6 +514,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..14b81dffce
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,410 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+#include <rte_vfio.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+
+#define SYSFS_AUXILIARY_DEVICES "/sys/bus/auxiliary/devices"
+
+int auxiliary_logtype_bus;
+
+const char *rte_auxiliary_get_sysfs_path(void)
+{
+	const char *path = NULL;
+
+#ifdef RTE_EXEC_ENV_LINUX
+	path = getenv("SYSFS_AUXILIARY_DEVICES");
+	if (path == NULL)
+		return SYSFS_AUXILIARY_DEVICES;
+#endif
+
+	return path;
+}
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+void
+auxiliary_on_scan(struct rte_auxiliary_device *dev)
+{
+	struct rte_devargs *devargs;
+
+	devargs = auxiliary_devargs_lookup(dev->name);
+	dev->device.devargs = devargs;
+}
+
+/*
+ * Match the auxiliary Driver and Device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
+		    const struct rte_auxiliary_device *auxiliary_dev)
+{
+	if (auxiliary_drv->match == NULL)
+		return false;
+	return auxiliary_drv->match(auxiliary_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	int ret;
+	bool dev_probed;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIAR_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+			dev->name, dev->device.numa_node);
+
+	/* no initialization when marked as blocked, return without error */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIAR_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIAR_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	dev_probed = rte_dev_is_probed(&dev->device);
+	if (dev_probed && !(dr->drv_flags & RTE_AUXILIARY_DRV_PROBE_AGAIN)) {
+		AUXILIAR_LOG(DEBUG, "  Device %s is already probed\n",
+				dev->device.name);
+		return -EEXIST;
+	}
+
+	AUXILIAR_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	if (!dev_probed) {
+		enum rte_iova_mode iova_mode;
+
+		iova_mode = rte_eal_iova_mode();
+		if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
+		    iova_mode != RTE_IOVA_VA) {
+			AUXILIAR_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+			return -EINVAL;
+		}
+
+		dev->driver = dr;
+	}
+
+	AUXILIAR_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+			dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (dev_probed)
+		return ret; /* no rollback if already succeeded earlier */
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_detach_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIAR_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		dev->name, dev->device.numa_node);
+
+	AUXILIAR_LOG(DEBUG, "  remove driver: %s %s\n",
+		dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	int rc = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIAR_LOG(ERR, "Requested device %s cannot be used\n",
+					dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	RTE_SET_USED(addr);
+	return auxiliary_exists(name) ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_auxiliary_dev,
+			    struct rte_auxiliary_device *new_auxiliary_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_auxiliary_dev, new_auxiliary_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_detach_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (!adev || !adev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (adev->driver->dma_map)
+		return adev->driver->dma_map(adev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (!adev || !adev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (adev->driver->dma_unmap)
+		return adev->driver->dma_unmap(adev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER(auxiliary_logtype_bus, bus.auxiliary, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..0e61020f3d
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	int ret;
+	const struct rte_kvargs *kvlist = _kvlist;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..7852e67b8c
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <rte_vfio.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+
+/**
+ * @file
+ * Linux auxiliary probing.
+ */
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	dev->device.bus = &auxiliary_bus.bus;
+	strlcpy(dev->name, name, sizeof(dev->name));
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 rte_auxiliary_get_sysfs_path(), name);
+	dir = opendir(rte_auxiliary_get_sysfs_path());
+	if (dir == NULL)
+		return true;
+	closedir(dir);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(rte_auxiliary_get_sysfs_path());
+	if (dir == NULL) {
+		AUXILIAR_LOG(ERR, "%s(): opendir failed: %s\n",
+			__func__, strerror(errno));
+		return -1;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+				rte_auxiliary_get_sysfs_path(), e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
+
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..0859620ed3
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Mellanox Technologies, Ltd
+
+if is_linux
+	headers = files('rte_bus_auxiliary.h')
+	sources = files('auxiliary_common.c',
+		'auxiliary_params.c',
+		'linux/auxiliary.c')
+	includes += include_directories('linux')
+	deps += ['kvargs']
+endif
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..b55827c74f
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_logtype_bus;
+
+#define AUXILIAR_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_logtype_bus, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+/**
+ * Test whether the auxiliary device exist
+ *
+ * @param name
+ *  Auxiliary device name
+ * @return
+ *  true on exists, false otherwise
+ */
+bool auxiliary_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Setup or update device when being scanned.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param auxiliary_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *auxiliary_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_auxiliary_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_auxiliary_dev
+ *	AUXILIARY device to be added before exist_auxiliary_dev
+ * @return void
+ */
+void auxiliary_insert_device(
+		struct rte_auxiliary_device *exist_auxiliary_dev,
+		struct rte_auxiliary_device *new_auxiliary_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param auxiliary_drv
+ *      auxiliary driver
+ * @param auxiliary_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
+		     const struct rte_auxiliary_device *auxiliary_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..b778a00749
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _RTE_BUS_AUXILIARY_H_
+#define _RTE_BUS_AUXILIARY_H_
+
+/**
+ * @file
+ *
+ * RTE Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ */
+typedef bool(auxiliary_match_t) (const char *);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ */
+typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
+				struct rte_auxiliary_device*);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ */
+typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
+				  uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+				    void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	auxiliary_match_t *match;            /**< Device match function. */
+	auxiliary_probe_t *probe;            /**< Device Probe function. */
+	auxiliary_remove_t *remove;          /**< Device Remove function. */
+	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
+	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+
+/** Device already probed can be probed again to check for new ports. */
+#define RTE_AUXILIARY_DRV_PROBE_AGAIN 0x0001
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Get sysfs pathname of auxiliary devices directory
+ *
+ * @return
+ *   Pathname of auxiliary devices directory
+ */
+__rte_experimental
+const char *rte_auxiliary_get_sysfs_path(void);
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&auxiliary_drv);		\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BUS_AUXILIARY_H_ */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..e012955b8f
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,11 @@
+DPDK_21 {
+	local: *;
+};
+
+EXPERIMENTAL {
+	global:
+
+	rte_auxiliary_get_sysfs_path;
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 2e7727af08..d581003f7d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
+drivers = ['auxiliary', 'dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
 std_deps = ['eal']
-- 
2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus
  2021-03-11 13:01 [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus Xueming Li
@ 2021-04-12  8:29 ` Xueming(Steven) Li
  2021-04-13  3:23 ` [dpdk-dev] [PATCH v1] " Xueming Li
  1 sibling, 0 replies; 28+ messages in thread
From: Xueming(Steven) Li @ 2021-04-12  8:29 UTC (permalink / raw)
  To: Xueming(Steven) Li, NBU-Contact-Thomas Monjalon
  Cc: dev, Parav Pandit, Ray Kinsella, Neil Horman


<snip>
> +/*
> + * Test whether the auxiliary device exist  */ bool
> +auxiliary_exists(const char *name) {
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +
> +	snprintf(dirname, sizeof(dirname), "%s/%s",
> +		 rte_auxiliary_get_sysfs_path(), name);
> +	dir = opendir(rte_auxiliary_get_sysfs_path());
> +	if (dir == NULL)
> +		return true;

Should return false here.

> +	closedir(dir);
> +	return false;

Return true, will fix in next version.

> +}
<snip>

^ permalink raw reply	[flat|nested] 28+ messages in thread

* [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-03-11 13:01 [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus Xueming Li
  2021-04-12  8:29 ` Xueming(Steven) Li
@ 2021-04-13  3:23 ` Xueming Li
  2021-04-13  8:49   ` Thomas Monjalon
                     ` (3 more replies)
  1 sibling, 4 replies; 28+ messages in thread
From: Xueming Li @ 2021-04-13  3:23 UTC (permalink / raw)
  To: Thomas Monjalon
  Cc: dev, xuemingl, Asaf Penso, Parav Pandit, Ray Kinsella, Neil Horman

Auxiliary [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary_device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 MAINTAINERS                               |   5 +
 drivers/bus/auxiliary/auxiliary_common.c  | 391 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
 drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
 drivers/bus/auxiliary/meson.build         |  17 +
 drivers/bus/auxiliary/private.h           | 118 +++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
 drivers/bus/auxiliary/version.map         |  10 +
 drivers/bus/meson.build                   |   2 +-
 9 files changed, 927 insertions(+), 1 deletion(-)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 0ec5588540..fd0e0c60ad 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -514,6 +514,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..bb9ba12be9
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,391 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+
+int auxiliary_bus_logtype;
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+void
+auxiliary_on_scan(struct rte_auxiliary_device *dev)
+{
+	struct rte_devargs *devargs;
+
+	devargs = auxiliary_devargs_lookup(dev->name);
+	dev->device.devargs = devargs;
+}
+
+/*
+ * Match the auxiliary Driver and Device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
+		    const struct rte_auxiliary_device *auxiliary_dev)
+{
+	if (auxiliary_drv->match == NULL)
+		return false;
+	return auxiliary_drv->match(auxiliary_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	int ret;
+	enum rte_iova_mode iova_mode;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	/* no initialization when marked as blocked, return without error */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+		return -EINVAL;
+	}
+
+	dev->driver = dr;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+		      dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
+		      dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	int rc = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	const char **out = addr;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (dr->match(name))
+			break;
+	}
+	if (dr != NULL && addr != NULL)
+		*out = name;
+	return dr != NULL ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_auxiliary_dev,
+			    struct rte_auxiliary_device *new_auxiliary_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_auxiliary_dev, new_auxiliary_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (!adev || !adev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (adev->driver->dma_map)
+		return adev->driver->dma_map(adev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (!adev || !adev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (adev->driver->dma_unmap)
+		return adev->driver->dma_unmap(adev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..5fc84b1953
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	int ret;
+	const struct rte_kvargs *kvlist = _kvlist;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..7888b6c5da
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/**
+ * @file
+ * Linux auxiliary probing.
+ */
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL)
+		return true;
+	closedir(dir);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(ERR, "%s(): opendir failed: %s\n",
+			      __func__, strerror(errno));
+		return -1;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..d84305acc9
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+if is_linux
+	headers = files('rte_bus_auxiliary.h')
+	sources = files('auxiliary_common.c',
+		'auxiliary_params.c',
+		'linux/auxiliary.c')
+	includes += include_directories('linux')
+	deps += ['kvargs']
+endif
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..b1b97bc26e
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+/**
+ * Test whether the auxiliary device exist
+ *
+ * @param name
+ *  Auxiliary device name
+ * @return
+ *  true on exists, false otherwise
+ */
+bool auxiliary_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Setup or update device when being scanned.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param auxiliary_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *auxiliary_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_auxiliary_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_auxiliary_dev
+ *	AUXILIARY device to be added before exist_auxiliary_dev
+ * @return void
+ */
+void auxiliary_insert_device(
+		struct rte_auxiliary_device *exist_auxiliary_dev,
+		struct rte_auxiliary_device *new_auxiliary_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param auxiliary_drv
+ *      auxiliary driver
+ * @param auxiliary_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
+		     const struct rte_auxiliary_device *auxiliary_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..816d65047b
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _RTE_BUS_AUXILIARY_H_
+#define _RTE_BUS_AUXILIARY_H_
+
+/**
+ * @file
+ *
+ * RTE Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ */
+typedef bool(auxiliary_match_t) (const char *);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ */
+typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
+				struct rte_auxiliary_device*);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ */
+typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
+				  uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+				    void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	auxiliary_match_t *match;            /**< Device match function. */
+	auxiliary_probe_t *probe;            /**< Device Probe function. */
+	auxiliary_remove_t *remove;          /**< Device Remove function. */
+	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
+	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&auxiliary_drv);		\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BUS_AUXILIARY_H_ */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..3d270baea7
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,10 @@
+DPDK_21 {
+	local: *;
+};
+
+EXPERIMENTAL {
+	global:
+
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 2e7727af08..d581003f7d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
+drivers = ['auxiliary', 'dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
 std_deps = ['eal']
-- 
2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-13  3:23 ` [dpdk-dev] [PATCH v1] " Xueming Li
@ 2021-04-13  8:49   ` Thomas Monjalon
  2021-04-14  2:59   ` Wang, Haiyue
                     ` (2 subsequent siblings)
  3 siblings, 0 replies; 28+ messages in thread
From: Thomas Monjalon @ 2021-04-13  8:49 UTC (permalink / raw)
  To: dev
  Cc: Xueming Li, Asaf Penso, Parav Pandit, Neil Horman, ferruh.yigit,
	andrew.rybchenko, jerinj, ajit.khaparde, haiyue.wang

I would like to Cc those who are interested in the auxiliary bus,
but it's difficult to know who will implement it.
Can we expect some reviews?


13/04/2021 05:23, Xueming Li:
> Auxiliary [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary_device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>





^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-13  3:23 ` [dpdk-dev] [PATCH v1] " Xueming Li
  2021-04-13  8:49   ` Thomas Monjalon
@ 2021-04-14  2:59   ` Wang, Haiyue
  2021-04-14  8:17     ` Thomas Monjalon
  2021-04-14 15:39     ` Xueming(Steven) Li
  2021-04-15  7:35   ` Wang, Haiyue
  2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
  3 siblings, 2 replies; 28+ messages in thread
From: Wang, Haiyue @ 2021-04-14  2:59 UTC (permalink / raw)
  To: Xueming Li, Thomas Monjalon
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, Neil Horman

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> Sent: Tuesday, April 13, 2021 11:23
> To: Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>;
> Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> Auxiliary [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary_device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> ---
>  MAINTAINERS                               |   5 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 391 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
>  drivers/bus/auxiliary/meson.build         |  17 +
>  drivers/bus/auxiliary/private.h           | 118 +++++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
>  drivers/bus/auxiliary/version.map         |  10 +
>  drivers/bus/meson.build                   |   2 +-
>  9 files changed, 927 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 


> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
> @@ -0,0 +1,391 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdbool.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <rte_errno.h>
> +#include <rte_interrupts.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_eal.h>
> +#include <rte_eal_paging.h>
> +#include <rte_string_fns.h>
> +#include <rte_common.h>
> +#include <rte_devargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +
> +int auxiliary_bus_logtype;
> +
> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name)
> +{
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
> +		if (strcmp(devargs->name, name) == 0)
> +			return devargs;
> +	}
> +	return NULL;
> +}
> +
> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_devargs *devargs;
> +
> +	devargs = auxiliary_devargs_lookup(dev->name);
> +	dev->device.devargs = devargs;

Can be simple as:

dev->device.devargs = auxiliary_devargs_lookup(dev->name);

> +}
> +
> +/*
> + * Match the auxiliary Driver and Device using driver function.
> + */
> +bool
> +auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
> +		    const struct rte_auxiliary_device *auxiliary_dev)

How about these auxiliary variable name style ?

const struct rte_auxiliary_driver *aux_drv,
const struct rte_auxiliary_device *aux_dev

> +{
> +	if (auxiliary_drv->match == NULL)
> +		return false;
> +	return auxiliary_drv->match(auxiliary_dev->name);
> +}
> +
> +/*
> + * Call the probe() function of the driver.
> + */
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> +			       struct rte_auxiliary_device *dev)
> +{
> +	int ret;
> +	enum rte_iova_mode iova_mode;
> +

RCT style ?
	enum rte_iova_mode iova_mode;
	int ret;


> +	if ((dr == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	/* The device is not blocked; Check if driver supports it */
> +	if (!auxiliary_match(dr, dev))
> +		/* Match of device and driver failed */
> +		return 1;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	/* no initialization when marked as blocked, return without error */
> +	if (dev->device.devargs != NULL &&
> +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> +		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
> +		return -1;
> +	}
> +
> +	if (dev->device.numa_node < 0) {
> +		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
> +		dev->device.numa_node = 0;
> +	}
> +
> +	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
> +
> +	iova_mode = rte_eal_iova_mode();
> +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&

'(dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)' should work, no need '> 0'

> +	    iova_mode != RTE_IOVA_VA) {
> +		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
> +		return -EINVAL;
> +	}
> +
> +	dev->driver = dr;
> +
> +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
> +		      dr->driver.name, dev->name, dev->device.numa_node);
> +	ret = dr->probe(dr, dev);
> +	if (ret)
> +		dev->driver = NULL;
> +	else
> +		dev->device.driver = &dr->driver;
> +
> +	return ret;
> +}
> +
> +/*
> + * Call the remove() function of the driver.
> + */
> +static int
> +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr;
> +	int ret = 0;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	dr = dev->driver;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
> +		      dev->name, dr->driver.name);
> +
> +	if (dr->remove) {
> +		ret = dr->remove(dev);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	/* clear driver structure */
> +	dev->driver = NULL;
> +	dev->device.driver = NULL;
> +
> +	return 0;
> +}
> +
> +/*
> + * Call the probe() function of all registered driver for the given device.
> + * Return < 0 if initialization failed.
> + * Return 1 if no driver is found for this device.
> + */
> +static int
> +auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr = NULL;
> +	int rc = 0;

These two variables need no initialization.

> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
> +		if (!dr->match(dev->name))
> +			continue;
> +
> +		rc = rte_auxiliary_probe_one_driver(dr, dev);
> +		if (rc < 0)
> +			/* negative value is an error */
> +			return rc;
> +		if (rc > 0)
> +			/* positive value means driver doesn't support it */
> +			continue;
> +		return 0;
> +	}
> +	return 1;
> +}
> +


> +static int
> +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> +{
> +	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);

How about to use 'aux_dev', instead of 'adev' ?

> +
> +	if (!adev || !adev->driver) {

' RTE_DEV_TO_AUXILIARY' is container of 'dev', so it should check 'dev != NULL',
not '!adev'. ; -)


> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (adev->driver->dma_map)
> +		return adev->driver->dma_map(adev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;
> +}
> +
> +static int
> +auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
> +		    size_t len)
> +{
> +	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (!adev || !adev->driver) {

The same comment as 'auxiliary_dma_map'

> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (adev->driver->dma_unmap)
> +		return adev->driver->dma_unmap(adev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;
> +}
> +


> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_params.c
> @@ -0,0 +1,58 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +
> +#include <rte_bus.h>
> +#include <rte_dev.h>
> +#include <rte_errno.h>
> +#include <rte_kvargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +enum auxiliary_params {
> +	RTE_AUXILIARY_PARAM_NAME,
> +};
> +
> +static const char * const auxiliary_params_keys[] = {
> +	[RTE_AUXILIARY_PARAM_NAME] = "name",
> +};
> +
> +static int
> +auxiliary_dev_match(const struct rte_device *dev,
> +	      const void *_kvlist)
> +{
> +	int ret;
> +	const struct rte_kvargs *kvlist = _kvlist;
> +

RCT.

> +	ret = rte_kvargs_process(kvlist,
> +			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
> +			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
> +
> +	return ret != 0 ? -1 : 0;
> +}
> +
> +void *
> +auxiliary_dev_iterate(const void *start,
> +		    const char *str,
> +		    const struct rte_dev_iterator *it __rte_unused)
> +{
> +	rte_bus_find_device_t find_device;
> +	struct rte_kvargs *kvargs = NULL;
> +	struct rte_device *dev;
> +
> +	if (str != NULL) {
> +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> +		if (kvargs == NULL) {
> +			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
> +			rte_errno = EINVAL;
> +			return NULL;
> +		}
> +	}
> +	find_device = auxiliary_bus.bus.find_device;
> +	dev = find_device(start, auxiliary_dev_match, kvargs);
> +	rte_kvargs_free(kvargs);
> +	return dev;
> +}


> diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
> new file mode 100644
> index 0000000000..7888b6c5da
> --- /dev/null
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
                              ^
                              |
Seems no need to add one more directory 'linux' layer, as the meson said "linux only".


> @@ -0,0 +1,147 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +#include <dirent.h>
> +
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <eal_filesystem.h>
> +
> +#include "../rte_bus_auxiliary.h"
> +#include "../private.h"
> +
> +#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
> +
> +/**
> + * @file
> + * Linux auxiliary probing.
> + */
> +
> +/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
> +static int
> +auxiliary_scan_one(const char *dirname, const char *name)
> +{
> +	struct rte_auxiliary_device *dev;
> +	struct rte_auxiliary_device *dev2;
> +	char filename[PATH_MAX];
> +	unsigned long tmp;
> +	int ret;
> +
> +	dev = malloc(sizeof(*dev));
> +	if (dev == NULL)
> +		return -1;
> +
> +	memset(dev, 0, sizeof(*dev));
> +	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
> +		free(dev);
> +		return -1;
> +	}
> +	dev->device.name = dev->name;
> +	dev->device.bus = &auxiliary_bus.bus;
> +
> +	/* Get numa node, default to 0 if not present */
> +	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
> +		 dirname, name);
> +	if (access(filename, F_OK) != -1) {
> +		if (eal_parse_sysfs_value(filename, &tmp) == 0)
> +			dev->device.numa_node = tmp;
> +		else
> +			dev->device.numa_node = -1;
> +	} else {
> +		dev->device.numa_node = 0;
> +	}
> +
> +	auxiliary_on_scan(dev);
> +
> +	/* Device is valid, add in list (sorted) */
> +	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
> +		ret = strcmp(dev->name, dev2->name);
> +		if (ret > 0)
> +			continue;
> +		if (ret < 0) {
> +			auxiliary_insert_device(dev2, dev);
> +		} else { /* already registered */
> +			if (rte_dev_is_probed(&dev2->device) &&
> +			    dev2->device.devargs != dev->device.devargs) {
> +				/* To probe device with new devargs. */
> +				rte_devargs_remove(dev2->device.devargs);
> +				auxiliary_on_scan(dev2);
> +			}
> +			free(dev);
> +		}
> +		return 0;
> +	}
> +	auxiliary_add_device(dev);
> +	return 0;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist
> + */
> +bool
> +auxiliary_exists(const char *name)

is_auxiliary_support() ?

> +{
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +
> +	snprintf(dirname, sizeof(dirname), "%s/%s",
> +		 AUXILIARY_SYSFS_PATH, name);
> +	dir = opendir(AUXILIARY_SYSFS_PATH);
> +	if (dir == NULL)
> +		return true;

false

> +	closedir(dir);
> +	return false;

true



> --
> 2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-14  2:59   ` Wang, Haiyue
@ 2021-04-14  8:17     ` Thomas Monjalon
  2021-04-14  8:30       ` Wang, Haiyue
  2021-04-14 15:49       ` Xueming(Steven) Li
  2021-04-14 15:39     ` Xueming(Steven) Li
  1 sibling, 2 replies; 28+ messages in thread
From: Thomas Monjalon @ 2021-04-14  8:17 UTC (permalink / raw)
  To: Xueming Li, Wang, Haiyue; +Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella

14/04/2021 04:59, Wang, Haiyue:
> From: Xueming Li
[...]
> > +void
> > +auxiliary_on_scan(struct rte_auxiliary_device *dev)
> > +{
> > +	struct rte_devargs *devargs;
> > +
> > +	devargs = auxiliary_devargs_lookup(dev->name);
> > +	dev->device.devargs = devargs;
> 
> Can be simple as:
> 
> dev->device.devargs = auxiliary_devargs_lookup(dev->name);
> 
> > +}
> > +
> > +/*
> > + * Match the auxiliary Driver and Device using driver function.
> > + */
> > +bool
> > +auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
> > +		    const struct rte_auxiliary_device *auxiliary_dev)
> 
> How about these auxiliary variable name style ?
> 
> const struct rte_auxiliary_driver *aux_drv,
> const struct rte_auxiliary_device *aux_dev

+1

[...]
> > +static int
> > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > +			       struct rte_auxiliary_device *dev)
> > +{
> > +	int ret;
> > +	enum rte_iova_mode iova_mode;
> > +
> 
> RCT style ?
> 	enum rte_iova_mode iova_mode;
> 	int ret;

I don't see the benefit of reverse christmas tree.

> > +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> 
> '(dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)' should work, no need '> 0'

Yes it's acceptable to consider bit testing as a boolean.

[...]
> > +static int
> > +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> > +{
> > +	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
> 
> How about to use 'aux_dev', instead of 'adev' ?
> 
> > +
> > +	if (!adev || !adev->driver) {
> 
> ' RTE_DEV_TO_AUXILIARY' is container of 'dev', so it should check 'dev != NULL',
> not '!adev'. ; -)

Yes and should be explicit NULL comparison.

[...]
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
>                               ^
>                               |
> Seems no need to add one more directory 'linux' layer, as the meson said "linux only".

I disagree.
Linux sub-directory is more explicit.
And who knows? There could be an implementation on other OSes in future.




^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-14  8:17     ` Thomas Monjalon
@ 2021-04-14  8:30       ` Wang, Haiyue
  2021-04-14 15:49       ` Xueming(Steven) Li
  1 sibling, 0 replies; 28+ messages in thread
From: Wang, Haiyue @ 2021-04-14  8:30 UTC (permalink / raw)
  To: Thomas Monjalon, Xueming Li; +Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, April 14, 2021 16:18
> To: Xueming Li <xuemingl@nvidia.com>; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>
> Subject: Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> 14/04/2021 04:59, Wang, Haiyue:
> > From: Xueming Li


> 
> [...]
> > > +static int
> > > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > > +			       struct rte_auxiliary_device *dev)
> > > +{
> > > +	int ret;
> > > +	enum rte_iova_mode iova_mode;
> > > +
> >
> > RCT style ?
> > 	enum rte_iova_mode iova_mode;
> > 	int ret;
> 
> I don't see the benefit of reverse christmas tree.
> 

I'm impacted by kernel code style ;-)


> > > +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> [...]
> > > --- /dev/null
> > > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> >                               ^
> >                               |
> > Seems no need to add one more directory 'linux' layer, as the meson said "linux only".
> 
> I disagree.
> Linux sub-directory is more explicit.
> And who knows? There could be an implementation on other OSes in future.

Make sense.

> 
> 


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-14  2:59   ` Wang, Haiyue
  2021-04-14  8:17     ` Thomas Monjalon
@ 2021-04-14 15:39     ` Xueming(Steven) Li
  2021-04-14 16:13       ` Wang, Haiyue
  1 sibling, 1 reply; 28+ messages in thread
From: Xueming(Steven) Li @ 2021-04-14 15:39 UTC (permalink / raw)
  To: Wang, Haiyue, NBU-Contact-Thomas Monjalon
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, Neil Horman

Hi Haiyue,

> > +
> > +/*
> > + * Test whether the auxiliary device exist  */ bool
> > +auxiliary_exists(const char *name)
> 
> is_auxiliary_support() ?

This function test whether an auxiliary device exists, name parameter is device name.
I think auxiliary_dev_exists(name) make more sense.

Agree with other comments, thanks!

^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-14  8:17     ` Thomas Monjalon
  2021-04-14  8:30       ` Wang, Haiyue
@ 2021-04-14 15:49       ` Xueming(Steven) Li
  1 sibling, 0 replies; 28+ messages in thread
From: Xueming(Steven) Li @ 2021-04-14 15:49 UTC (permalink / raw)
  To: NBU-Contact-Thomas Monjalon, Wang, Haiyue
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella



> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, April 14, 2021 4:18 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>
> Subject: Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> 14/04/2021 04:59, Wang, Haiyue:
> > From: Xueming Li
> [...]
> > > +void
> > > +auxiliary_on_scan(struct rte_auxiliary_device *dev) {
> > > +	struct rte_devargs *devargs;
> > > +
> > > +	devargs = auxiliary_devargs_lookup(dev->name);
> > > +	dev->device.devargs = devargs;
> >
> > Can be simple as:
> >
> > dev->device.devargs = auxiliary_devargs_lookup(dev->name);
> >
> > > +}
> > > +
> > > +/*
> > > + * Match the auxiliary Driver and Device using driver function.
> > > + */
> > > +bool
> > > +auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
> > > +		    const struct rte_auxiliary_device *auxiliary_dev)
> >
> > How about these auxiliary variable name style ?
> >
> > const struct rte_auxiliary_driver *aux_drv, const struct
> > rte_auxiliary_device *aux_dev
> 
> +1
> 
> [...]
> > > +static int
> > > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > > +			       struct rte_auxiliary_device *dev) {
> > > +	int ret;
> > > +	enum rte_iova_mode iova_mode;
> > > +
> >
> > RCT style ?
> > 	enum rte_iova_mode iova_mode;
> > 	int ret;
> 
> I don't see the benefit of reverse christmas tree.
> 
> > > +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> >
> > '(dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)' should work, no need '> 0'
> 
> Yes it's acceptable to consider bit testing as a boolean.
> 
> [...]
> > > +static int
> > > +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t
> > > +iova, size_t len) {
> > > +	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
> >
> > How about to use 'aux_dev', instead of 'adev' ?
> >
> > > +
> > > +	if (!adev || !adev->driver) {
> >
> > ' RTE_DEV_TO_AUXILIARY' is container of 'dev', so it should check 'dev
> > != NULL', not '!adev'. ; -)
> 
> Yes and should be explicit NULL comparison.
> 
> [...]
> > > --- /dev/null
> > > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> >                               ^
> >                               |
> > Seems no need to add one more directory 'linux' layer, as the meson said "linux only".
> 
> I disagree.
> Linux sub-directory is more explicit.
> And who knows? There could be an implementation on other OSes in future.

This reminds me to change meson, allows bus always available to avoid compilation error,
also need to add stubs with __rte_weak for all functions in common file.

> 
> 


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-14 15:39     ` Xueming(Steven) Li
@ 2021-04-14 16:13       ` Wang, Haiyue
  0 siblings, 0 replies; 28+ messages in thread
From: Wang, Haiyue @ 2021-04-14 16:13 UTC (permalink / raw)
  To: Xueming(Steven) Li, NBU-Contact-Thomas Monjalon
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, Neil Horman

> -----Original Message-----
> From: Xueming(Steven) Li <xuemingl@nvidia.com>
> Sent: Wednesday, April 14, 2021 23:39
> To: Wang, Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> Hi Haiyue,
> 
> > > +
> > > +/*
> > > + * Test whether the auxiliary device exist  */ bool
> > > +auxiliary_exists(const char *name)
> >
> > is_auxiliary_support() ?
> 
> This function test whether an auxiliary device exists, name parameter is device name.
> I think auxiliary_dev_exists(name) make more sense.

More clear now. ;-)

> 
> Agree with other comments, thanks!

^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-13  3:23 ` [dpdk-dev] [PATCH v1] " Xueming Li
  2021-04-13  8:49   ` Thomas Monjalon
  2021-04-14  2:59   ` Wang, Haiyue
@ 2021-04-15  7:35   ` Wang, Haiyue
  2021-04-15  7:46     ` Xueming(Steven) Li
  2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
  3 siblings, 1 reply; 28+ messages in thread
From: Wang, Haiyue @ 2021-04-15  7:35 UTC (permalink / raw)
  To: Xueming Li, Thomas Monjalon
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, Neil Horman

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> Sent: Tuesday, April 13, 2021 11:23
> To: Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>;
> Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> Auxiliary [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary_device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> ---
>  MAINTAINERS                               |   5 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 391 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
>  drivers/bus/auxiliary/meson.build         |  17 +
>  drivers/bus/auxiliary/private.h           | 118 +++++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
>  drivers/bus/auxiliary/version.map         |  10 +
>  drivers/bus/meson.build                   |   2 +-
>  9 files changed, 927 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 


> --- /dev/null
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> @@ -0,0 +1,180 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef _RTE_BUS_AUXILIARY_H_
> +#define _RTE_BUS_AUXILIARY_H_
> +
> +/**
> + * @file
> + *
> + * RTE Auxiliary Bus Interface.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <sys/queue.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +
> +#include <rte_debug.h>
> +#include <rte_interrupts.h>
> +#include <rte_dev.h>
> +#include <rte_bus.h>
> +#include <rte_kvargs.h>
> +
> +/* Forward declarations */
> +struct rte_auxiliary_driver;
> +struct rte_auxiliary_bus;
> +struct rte_auxiliary_device;
> +
> +/**
> + * Match function for the driver to decide if device can be handled.
> + */
> +typedef bool(auxiliary_match_t) (const char *);
> +
> +/**
> + * Initialization function for the driver called during auxiliary probing.
> + */
> +typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
> +				struct rte_auxiliary_device*);
> +
> +/**
> + * Uninitialization function for the driver called during hotplugging.
> + */
> +typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
> +
> +/**
> + * Driver-specific DMA mapping. After a successful call the device
> + * will be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be mapped.
> + * @param iova
> + *   Starting IOVA address of memory to be mapped.
> + * @param len
> + *   Length of memory segment being mapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
> +				  uint64_t iova, size_t len);
> +
> +/**
> + * Driver-specific DMA un-mapping. After a successful call the device
> + * will not be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be unmapped.
> + * @param iova
> + *   Starting IOVA address of memory to be unmapped.
> + * @param len
> + *   Length of memory segment being unmapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> +				    void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * A structure describing an auxiliary device.
> + */
> +struct rte_auxiliary_device {
> +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> +	struct rte_device device;                 /**< Inherit core device */
> +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> +};
> +
> +/** List of auxiliary devices */
> +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> +/** List of auxiliary drivers */
> +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> +
> +/**
> + * Structure describing the auxiliary bus
> + */
> +struct rte_auxiliary_bus {
> +	struct rte_bus bus;                  /**< Inherit the generic class */
> +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
> +};
> +
> +/**
> + * A structure describing an auxiliary driver.
> + */
> +struct rte_auxiliary_driver {
> +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> +	struct rte_driver driver;            /**< Inherit core driver. */
> +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> +	auxiliary_match_t *match;            /**< Device match function. */
> +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */

These API type can be pointer type defined, then no need "*":

typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
				    void *addr, uint64_t iova, size_t len);

auxiliary_dma_unmap_t dma_unmap;

Like:
https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759-1-thomas@monjalon.net/

typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
				 void *addr, uint64_t iova, size_t len);

> --
> 2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-15  7:35   ` Wang, Haiyue
@ 2021-04-15  7:46     ` Xueming(Steven) Li
  2021-04-15  7:51       ` Wang, Haiyue
  0 siblings, 1 reply; 28+ messages in thread
From: Xueming(Steven) Li @ 2021-04-15  7:46 UTC (permalink / raw)
  To: Wang, Haiyue, NBU-Contact-Thomas Monjalon
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, Neil Horman


> -----Original Message-----
> From: Wang, Haiyue <haiyue.wang@intel.com>
> Sent: Thursday, April 15, 2021 3:36 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman
> <nhorman@tuxdriver.com>
> Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > Sent: Tuesday, April 13, 2021 11:23
> > To: Thomas Monjalon <thomas@monjalon.net>
> > Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso <asafp@nvidia.com>;
> > Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil
> > Horman <nhorman@tuxdriver.com>
> > Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> >
> > Auxiliary [1] provides a way to split function into child-devices
> > representing sub-domains of functionality. Each auxiliary_device
> > represents a part of its parent functionality.
> >
> > Auxiliary device is identified by unique device name, sysfs path:
> >   /sys/bus/auxiliary/devices/<name>
> >
> > [1] kernel auxiliary bus document:
> > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > ---
> >  MAINTAINERS                               |   5 +
> >  drivers/bus/auxiliary/auxiliary_common.c  | 391
> > ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
> >  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
> >  drivers/bus/auxiliary/meson.build         |  17 +
> >  drivers/bus/auxiliary/private.h           | 118 +++++++
> >  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
> >  drivers/bus/auxiliary/version.map         |  10 +
> >  drivers/bus/meson.build                   |   2 +-
> >  9 files changed, 927 insertions(+), 1 deletion(-)  create mode 100644
> > drivers/bus/auxiliary/auxiliary_common.c
> >  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> >  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> >  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> > 100644 drivers/bus/auxiliary/private.h  create mode 100644
> > drivers/bus/auxiliary/rte_bus_auxiliary.h
> >  create mode 100644 drivers/bus/auxiliary/version.map
> >
> 
> 
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > @@ -0,0 +1,180 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2021 Mellanox Technologies, Ltd  */
> > +
> > +#ifndef _RTE_BUS_AUXILIARY_H_
> > +#define _RTE_BUS_AUXILIARY_H_
> > +
> > +/**
> > + * @file
> > + *
> > + * RTE Auxiliary Bus Interface.
> > + */
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <limits.h>
> > +#include <errno.h>
> > +#include <sys/queue.h>
> > +#include <stdint.h>
> > +#include <inttypes.h>
> > +
> > +#include <rte_debug.h>
> > +#include <rte_interrupts.h>
> > +#include <rte_dev.h>
> > +#include <rte_bus.h>
> > +#include <rte_kvargs.h>
> > +
> > +/* Forward declarations */
> > +struct rte_auxiliary_driver;
> > +struct rte_auxiliary_bus;
> > +struct rte_auxiliary_device;
> > +
> > +/**
> > + * Match function for the driver to decide if device can be handled.
> > + */
> > +typedef bool(auxiliary_match_t) (const char *);
> > +
> > +/**
> > + * Initialization function for the driver called during auxiliary probing.
> > + */
> > +typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
> > +				struct rte_auxiliary_device*);
> > +
> > +/**
> > + * Uninitialization function for the driver called during hotplugging.
> > + */
> > +typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
> > +
> > +/**
> > + * Driver-specific DMA mapping. After a successful call the device
> > + * will be able to read/write from/to this segment.
> > + *
> > + * @param dev
> > + *   Pointer to the auxiliary device.
> > + * @param addr
> > + *   Starting virtual address of memory to be mapped.
> > + * @param iova
> > + *   Starting IOVA address of memory to be mapped.
> > + * @param len
> > + *   Length of memory segment being mapped.
> > + * @return
> > + *   - 0 On success.
> > + *   - Negative value and rte_errno is set otherwise.
> > + */
> > +typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
> > +				  uint64_t iova, size_t len);
> > +
> > +/**
> > + * Driver-specific DMA un-mapping. After a successful call the device
> > + * will not be able to read/write from/to this segment.
> > + *
> > + * @param dev
> > + *   Pointer to the auxiliary device.
> > + * @param addr
> > + *   Starting virtual address of memory to be unmapped.
> > + * @param iova
> > + *   Starting IOVA address of memory to be unmapped.
> > + * @param len
> > + *   Length of memory segment being unmapped.
> > + * @return
> > + *   - 0 On success.
> > + *   - Negative value and rte_errno is set otherwise.
> > + */
> > +typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > +				    void *addr, uint64_t iova, size_t len);
> > +
> > +/**
> > + * A structure describing an auxiliary device.
> > + */
> > +struct rte_auxiliary_device {
> > +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> > +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> > +	struct rte_device device;                 /**< Inherit core device */
> > +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> > +};
> > +
> > +/** List of auxiliary devices */
> > +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> > +/** List of auxiliary drivers */
> > +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> > +
> > +/**
> > + * Structure describing the auxiliary bus  */ struct
> > +rte_auxiliary_bus {
> > +	struct rte_bus bus;                  /**< Inherit the generic class */
> > +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> > +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers
> > +*/ };
> > +
> > +/**
> > + * A structure describing an auxiliary driver.
> > + */
> > +struct rte_auxiliary_driver {
> > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > +	struct rte_driver driver;            /**< Inherit core driver. */
> > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > +	auxiliary_match_t *match;            /**< Device match function. */
> > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> 
> These API type can be pointer type defined, then no need "*":
> 
> typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> 				    void *addr, uint64_t iova, size_t len);
> 
> auxiliary_dma_unmap_t dma_unmap;
> 
> Like:
> https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759-1-thomas@monjalon.net/
> 
> typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> 				 void *addr, uint64_t iova, size_t len);

Thanks, is there a reason to prefer pointer type? 

Thoma's patch looks good, will rebase on it once accepted.

> 
> > --
> > 2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-15  7:46     ` Xueming(Steven) Li
@ 2021-04-15  7:51       ` Wang, Haiyue
  2021-04-15  7:55         ` Xueming(Steven) Li
  0 siblings, 1 reply; 28+ messages in thread
From: Wang, Haiyue @ 2021-04-15  7:51 UTC (permalink / raw)
  To: Xueming(Steven) Li, NBU-Contact-Thomas Monjalon
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, Neil Horman

> -----Original Message-----
> From: Xueming(Steven) Li <xuemingl@nvidia.com>
> Sent: Thursday, April 15, 2021 15:46
> To: Wang, Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> 
> > -----Original Message-----
> > From: Wang, Haiyue <haiyue.wang@intel.com>
> > Sent: Thursday, April 15, 2021 3:36 PM
> > To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> > Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; Neil Horman
> > <nhorman@tuxdriver.com>
> > Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> >
> > > -----Original Message-----
> > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > Sent: Tuesday, April 13, 2021 11:23
> > > To: Thomas Monjalon <thomas@monjalon.net>
> > > Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso <asafp@nvidia.com>;
> > > Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil
> > > Horman <nhorman@tuxdriver.com>
> > > Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> > >
> > > Auxiliary [1] provides a way to split function into child-devices
> > > representing sub-domains of functionality. Each auxiliary_device
> > > represents a part of its parent functionality.
> > >
> > > Auxiliary device is identified by unique device name, sysfs path:
> > >   /sys/bus/auxiliary/devices/<name>
> > >
> > > [1] kernel auxiliary bus document:
> > > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> > >
> > > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > > ---
> > >  MAINTAINERS                               |   5 +
> > >  drivers/bus/auxiliary/auxiliary_common.c  | 391
> > > ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
> > >  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
> > >  drivers/bus/auxiliary/meson.build         |  17 +
> > >  drivers/bus/auxiliary/private.h           | 118 +++++++
> > >  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
> > >  drivers/bus/auxiliary/version.map         |  10 +
> > >  drivers/bus/meson.build                   |   2 +-
> > >  9 files changed, 927 insertions(+), 1 deletion(-)  create mode 100644
> > > drivers/bus/auxiliary/auxiliary_common.c
> > >  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> > >  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> > >  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> > > 100644 drivers/bus/auxiliary/private.h  create mode 100644
> > > drivers/bus/auxiliary/rte_bus_auxiliary.h
> > >  create mode 100644 drivers/bus/auxiliary/version.map
> > >
> >
> >
> > > --- /dev/null
> > > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > > @@ -0,0 +1,180 @@
> > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > + * Copyright 2021 Mellanox Technologies, Ltd  */
> > > +
> > > +#ifndef _RTE_BUS_AUXILIARY_H_
> > > +#define _RTE_BUS_AUXILIARY_H_
> > > +
> > > +/**
> > > + * @file
> > > + *
> > > + * RTE Auxiliary Bus Interface.
> > > + */
> > > +
> > > +#ifdef __cplusplus
> > > +extern "C" {
> > > +#endif
> > > +
> > > +#include <stdio.h>
> > > +#include <stdlib.h>
> > > +#include <limits.h>
> > > +#include <errno.h>
> > > +#include <sys/queue.h>
> > > +#include <stdint.h>
> > > +#include <inttypes.h>
> > > +
> > > +#include <rte_debug.h>
> > > +#include <rte_interrupts.h>
> > > +#include <rte_dev.h>
> > > +#include <rte_bus.h>
> > > +#include <rte_kvargs.h>
> > > +
> > > +/* Forward declarations */
> > > +struct rte_auxiliary_driver;
> > > +struct rte_auxiliary_bus;
> > > +struct rte_auxiliary_device;
> > > +
> > > +/**
> > > + * Match function for the driver to decide if device can be handled.
> > > + */
> > > +typedef bool(auxiliary_match_t) (const char *);
> > > +
> > > +/**
> > > + * Initialization function for the driver called during auxiliary probing.
> > > + */
> > > +typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
> > > +				struct rte_auxiliary_device*);
> > > +
> > > +/**
> > > + * Uninitialization function for the driver called during hotplugging.
> > > + */
> > > +typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
> > > +
> > > +/**
> > > + * Driver-specific DMA mapping. After a successful call the device
> > > + * will be able to read/write from/to this segment.
> > > + *
> > > + * @param dev
> > > + *   Pointer to the auxiliary device.
> > > + * @param addr
> > > + *   Starting virtual address of memory to be mapped.
> > > + * @param iova
> > > + *   Starting IOVA address of memory to be mapped.
> > > + * @param len
> > > + *   Length of memory segment being mapped.
> > > + * @return
> > > + *   - 0 On success.
> > > + *   - Negative value and rte_errno is set otherwise.
> > > + */
> > > +typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
> > > +				  uint64_t iova, size_t len);
> > > +
> > > +/**
> > > + * Driver-specific DMA un-mapping. After a successful call the device
> > > + * will not be able to read/write from/to this segment.
> > > + *
> > > + * @param dev
> > > + *   Pointer to the auxiliary device.
> > > + * @param addr
> > > + *   Starting virtual address of memory to be unmapped.
> > > + * @param iova
> > > + *   Starting IOVA address of memory to be unmapped.
> > > + * @param len
> > > + *   Length of memory segment being unmapped.
> > > + * @return
> > > + *   - 0 On success.
> > > + *   - Negative value and rte_errno is set otherwise.
> > > + */
> > > +typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > +				    void *addr, uint64_t iova, size_t len);
> > > +
> > > +/**
> > > + * A structure describing an auxiliary device.
> > > + */
> > > +struct rte_auxiliary_device {
> > > +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> > > +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> > > +	struct rte_device device;                 /**< Inherit core device */
> > > +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> > > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> > > +};
> > > +
> > > +/** List of auxiliary devices */
> > > +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> > > +/** List of auxiliary drivers */
> > > +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> > > +
> > > +/**
> > > + * Structure describing the auxiliary bus  */ struct
> > > +rte_auxiliary_bus {
> > > +	struct rte_bus bus;                  /**< Inherit the generic class */
> > > +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> > > +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers
> > > +*/ };
> > > +
> > > +/**
> > > + * A structure describing an auxiliary driver.
> > > + */
> > > +struct rte_auxiliary_driver {
> > > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > > +	struct rte_driver driver;            /**< Inherit core driver. */
> > > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > > +	auxiliary_match_t *match;            /**< Device match function. */
> > > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> >
> > These API type can be pointer type defined, then no need "*":
> >
> > typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > 				    void *addr, uint64_t iova, size_t len);
> >
> > auxiliary_dma_unmap_t dma_unmap;
> >
> > Like:
> > https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759-1-thomas@monjalon.net/
> >
> > typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> > 				 void *addr, uint64_t iova, size_t len);
> 
> Thanks, is there a reason to prefer pointer type?

Good practice to make code beautiful ? ;-)

> 
> Thoma's patch looks good, will rebase on it once accepted.

I mean the function type is defined as pointer type.

> 
> >
> > > --
> > > 2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-15  7:51       ` Wang, Haiyue
@ 2021-04-15  7:55         ` Xueming(Steven) Li
  2021-04-15  7:59           ` Thomas Monjalon
  0 siblings, 1 reply; 28+ messages in thread
From: Xueming(Steven) Li @ 2021-04-15  7:55 UTC (permalink / raw)
  To: Wang, Haiyue, NBU-Contact-Thomas Monjalon
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, Neil Horman


> -----Original Message-----
> From: Wang, Haiyue <haiyue.wang@intel.com>
> Sent: Thursday, April 15, 2021 3:52 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman
> <nhorman@tuxdriver.com>
> Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> > -----Original Message-----
> > From: Xueming(Steven) Li <xuemingl@nvidia.com>
> > Sent: Thursday, April 15, 2021 15:46
> > To: Wang, Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon
> > <thomas@monjalon.net>
> > Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit
> > <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman
> > <nhorman@tuxdriver.com>
> > Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary
> > bus
> >
> >
> > > -----Original Message-----
> > > From: Wang, Haiyue <haiyue.wang@intel.com>
> > > Sent: Thursday, April 15, 2021 3:36 PM
> > > To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas
> > > Monjalon <thomas@monjalon.net>
> > > Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit
> > > <parav@nvidia.com>; Ray Kinsella
> > <mdr@ashroe.eu>; Neil Horman
> > > <nhorman@tuxdriver.com>
> > > Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce
> > > auxiliary bus
> > >
> > > > -----Original Message-----
> > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > > Sent: Tuesday, April 13, 2021 11:23
> > > > To: Thomas Monjalon <thomas@monjalon.net>
> > > > Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso
> > > > <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> > > > <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> > > > Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary
> > > > bus
> > > >
> > > > Auxiliary [1] provides a way to split function into child-devices
> > > > representing sub-domains of functionality. Each auxiliary_device
> > > > represents a part of its parent functionality.
> > > >
> > > > Auxiliary device is identified by unique device name, sysfs path:
> > > >   /sys/bus/auxiliary/devices/<name>
> > > >
> > > > [1] kernel auxiliary bus document:
> > > > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.ht
> > > > ml
> > > >
> > > > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > > > ---
> > > >  MAINTAINERS                               |   5 +
> > > >  drivers/bus/auxiliary/auxiliary_common.c  | 391
> > > > ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c
> > > > ++++++++++++++++++++++ |  58 ++++
> > > >  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
> > > >  drivers/bus/auxiliary/meson.build         |  17 +
> > > >  drivers/bus/auxiliary/private.h           | 118 +++++++
> > > >  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
> > > >  drivers/bus/auxiliary/version.map         |  10 +
> > > >  drivers/bus/meson.build                   |   2 +-
> > > >  9 files changed, 927 insertions(+), 1 deletion(-)  create mode
> > > > 100644 drivers/bus/auxiliary/auxiliary_common.c
> > > >  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> > > >  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> > > >  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> > > > 100644 drivers/bus/auxiliary/private.h  create mode 100644
> > > > drivers/bus/auxiliary/rte_bus_auxiliary.h
> > > >  create mode 100644 drivers/bus/auxiliary/version.map
> > > >
> > >
> > >
> > > > --- /dev/null
> > > > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > > > @@ -0,0 +1,180 @@
> > > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > > + * Copyright 2021 Mellanox Technologies, Ltd  */
> > > > +
> > > > +#ifndef _RTE_BUS_AUXILIARY_H_
> > > > +#define _RTE_BUS_AUXILIARY_H_
> > > > +
> > > > +/**
> > > > + * @file
> > > > + *
> > > > + * RTE Auxiliary Bus Interface.
> > > > + */
> > > > +
> > > > +#ifdef __cplusplus
> > > > +extern "C" {
> > > > +#endif
> > > > +
> > > > +#include <stdio.h>
> > > > +#include <stdlib.h>
> > > > +#include <limits.h>
> > > > +#include <errno.h>
> > > > +#include <sys/queue.h>
> > > > +#include <stdint.h>
> > > > +#include <inttypes.h>
> > > > +
> > > > +#include <rte_debug.h>
> > > > +#include <rte_interrupts.h>
> > > > +#include <rte_dev.h>
> > > > +#include <rte_bus.h>
> > > > +#include <rte_kvargs.h>
> > > > +
> > > > +/* Forward declarations */
> > > > +struct rte_auxiliary_driver;
> > > > +struct rte_auxiliary_bus;
> > > > +struct rte_auxiliary_device;
> > > > +
> > > > +/**
> > > > + * Match function for the driver to decide if device can be handled.
> > > > + */
> > > > +typedef bool(auxiliary_match_t) (const char *);
> > > > +
> > > > +/**
> > > > + * Initialization function for the driver called during auxiliary probing.
> > > > + */
> > > > +typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
> > > > +				struct rte_auxiliary_device*);
> > > > +
> > > > +/**
> > > > + * Uninitialization function for the driver called during hotplugging.
> > > > + */
> > > > +typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
> > > > +
> > > > +/**
> > > > + * Driver-specific DMA mapping. After a successful call the
> > > > +device
> > > > + * will be able to read/write from/to this segment.
> > > > + *
> > > > + * @param dev
> > > > + *   Pointer to the auxiliary device.
> > > > + * @param addr
> > > > + *   Starting virtual address of memory to be mapped.
> > > > + * @param iova
> > > > + *   Starting IOVA address of memory to be mapped.
> > > > + * @param len
> > > > + *   Length of memory segment being mapped.
> > > > + * @return
> > > > + *   - 0 On success.
> > > > + *   - Negative value and rte_errno is set otherwise.
> > > > + */
> > > > +typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
> > > > +				  uint64_t iova, size_t len);
> > > > +
> > > > +/**
> > > > + * Driver-specific DMA un-mapping. After a successful call the
> > > > +device
> > > > + * will not be able to read/write from/to this segment.
> > > > + *
> > > > + * @param dev
> > > > + *   Pointer to the auxiliary device.
> > > > + * @param addr
> > > > + *   Starting virtual address of memory to be unmapped.
> > > > + * @param iova
> > > > + *   Starting IOVA address of memory to be unmapped.
> > > > + * @param len
> > > > + *   Length of memory segment being unmapped.
> > > > + * @return
> > > > + *   - 0 On success.
> > > > + *   - Negative value and rte_errno is set otherwise.
> > > > + */
> > > > +typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > > +				    void *addr, uint64_t iova, size_t len);
> > > > +
> > > > +/**
> > > > + * A structure describing an auxiliary device.
> > > > + */
> > > > +struct rte_auxiliary_device {
> > > > +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> > > > +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> > > > +	struct rte_device device;                 /**< Inherit core device */
> > > > +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> > > > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> > > > +};
> > > > +
> > > > +/** List of auxiliary devices */
> > > > +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> > > > +/** List of auxiliary drivers */
> > > > +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> > > > +
> > > > +/**
> > > > + * Structure describing the auxiliary bus  */ struct
> > > > +rte_auxiliary_bus {
> > > > +	struct rte_bus bus;                  /**< Inherit the generic class */
> > > > +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> > > > +	struct rte_auxiliary_driver_list driver_list;  /**< List of
> > > > +drivers */ };
> > > > +
> > > > +/**
> > > > + * A structure describing an auxiliary driver.
> > > > + */
> > > > +struct rte_auxiliary_driver {
> > > > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > > > +	struct rte_driver driver;            /**< Inherit core driver. */
> > > > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > > > +	auxiliary_match_t *match;            /**< Device match function. */
> > > > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > > > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > > > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > > > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> > >
> > > These API type can be pointer type defined, then no need "*":
> > >
> > > typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > 				    void *addr, uint64_t iova, size_t len);
> > >
> > > auxiliary_dma_unmap_t dma_unmap;
> > >
> > > Like:
> > > https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759
> > > -1-thomas@monjalon.net/
> > >
> > > typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> > > 				 void *addr, uint64_t iova, size_t len);
> >
> > Thanks, is there a reason to prefer pointer type?
> 
> Good practice to make code beautiful ? ;-)
> 
> >
> > Thoma's patch looks good, will rebase on it once accepted.
> 
> I mean the function type is defined as pointer type.
Yes, I'm talking another topic :)
His patch makes dma map/unmap higher level to rte_bus, so that no need to define the api again here.

> 
> >
> > >
> > > > --
> > > > 2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-15  7:55         ` Xueming(Steven) Li
@ 2021-04-15  7:59           ` Thomas Monjalon
  2021-04-15  8:06             ` Wang, Haiyue
  0 siblings, 1 reply; 28+ messages in thread
From: Thomas Monjalon @ 2021-04-15  7:59 UTC (permalink / raw)
  To: Wang, Haiyue, Xueming(Steven) Li
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, david.marchand,
	ferruh.yigit, andrew.rybchenko, olivier.matz

15/04/2021 09:55, Xueming(Steven) Li:
> From: Wang, Haiyue <haiyue.wang@intel.com>
> > From: Xueming(Steven) Li <xuemingl@nvidia.com>
> > > From: Wang, Haiyue <haiyue.wang@intel.com>
> > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > > > +/**
> > > > > + * A structure describing an auxiliary driver.
> > > > > + */
> > > > > +struct rte_auxiliary_driver {
> > > > > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > > > > +	struct rte_driver driver;            /**< Inherit core driver. */
> > > > > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > > > > +	auxiliary_match_t *match;            /**< Device match function. */
> > > > > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > > > > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > > > > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > > > > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> > > >
> > > > These API type can be pointer type defined, then no need "*":
> > > >
> > > > typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > > 				    void *addr, uint64_t iova, size_t len);
> > > >
> > > > auxiliary_dma_unmap_t dma_unmap;
> > > >
> > > > Like:
> > > > https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759
> > > > -1-thomas@monjalon.net/
> > > >
> > > > typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> > > > 				 void *addr, uint64_t iova, size_t len);
> > >
> > > Thanks, is there a reason to prefer pointer type?
> > 
> > Good practice to make code beautiful ? ;-)

Honestly, I am not sure which one is better,
having the pointer type hidden in the typedef or explicit?

> > > Thoma's patch looks good, will rebase on it once accepted.
> > 
> > I mean the function type is defined as pointer type.
> Yes, I'm talking another topic :)
> His patch makes dma map/unmap higher level to rte_bus, so that no need to define the api again here.

I think my patch will be abandoned.



^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
  2021-04-15  7:59           ` Thomas Monjalon
@ 2021-04-15  8:06             ` Wang, Haiyue
  0 siblings, 0 replies; 28+ messages in thread
From: Wang, Haiyue @ 2021-04-15  8:06 UTC (permalink / raw)
  To: Thomas Monjalon, Xueming(Steven) Li
  Cc: dev, Asaf Penso, Parav Pandit, Ray Kinsella, david.marchand,
	Yigit, Ferruh, andrew.rybchenko, olivier.matz

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Thursday, April 15, 2021 15:59
> To: Wang, Haiyue <haiyue.wang@intel.com>; Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; david.marchand@redhat.com; Yigit, Ferruh <ferruh.yigit@intel.com>;
> andrew.rybchenko@oktetlabs.ru; olivier.matz@6wind.com
> Subject: Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> 15/04/2021 09:55, Xueming(Steven) Li:
> > From: Wang, Haiyue <haiyue.wang@intel.com>
> > > From: Xueming(Steven) Li <xuemingl@nvidia.com>
> > > > From: Wang, Haiyue <haiyue.wang@intel.com>
> > > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > > > > +/**
> > > > > > + * A structure describing an auxiliary driver.
> > > > > > + */
> > > > > > +struct rte_auxiliary_driver {
> > > > > > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > > > > > +	struct rte_driver driver;            /**< Inherit core driver. */
> > > > > > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > > > > > +	auxiliary_match_t *match;            /**< Device match function. */
> > > > > > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > > > > > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > > > > > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > > > > > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> > > > >
> > > > > These API type can be pointer type defined, then no need "*":
> > > > >
> > > > > typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > > > 				    void *addr, uint64_t iova, size_t len);
> > > > >
> > > > > auxiliary_dma_unmap_t dma_unmap;
> > > > >
> > > > > Like:
> > > > > https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759
> > > > > -1-thomas@monjalon.net/
> > > > >
> > > > > typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> > > > > 				 void *addr, uint64_t iova, size_t len);
> > > >
> > > > Thanks, is there a reason to prefer pointer type?
> > >
> > > Good practice to make code beautiful ? ;-)
> 
> Honestly, I am not sure which one is better,
> having the pointer type hidden in the typedef or explicit?

I got this style from 'struct eth_dev_ops', Steven can make the final decision,
just a suggestion.

> 
> > > > Thoma's patch looks good, will rebase on it once accepted.
> > >
> > > I mean the function type is defined as pointer type.
> > Yes, I'm talking another topic :)
> > His patch makes dma map/unmap higher level to rte_bus, so that no need to define the api again here.
> 
> I think my patch will be abandoned.
> 


^ permalink raw reply	[flat|nested] 28+ messages in thread

* [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
  2021-04-13  3:23 ` [dpdk-dev] [PATCH v1] " Xueming Li
                     ` (2 preceding siblings ...)
  2021-04-15  7:35   ` Wang, Haiyue
@ 2021-05-10 13:47   ` Xueming Li
  2021-05-11  9:47     ` Kinsella, Ray
                       ` (5 more replies)
  3 siblings, 6 replies; 28+ messages in thread
From: Xueming Li @ 2021-05-10 13:47 UTC (permalink / raw)
  To: NBU-Contact-Thomas Monjalon
  Cc: dev, xuemingl, Parav Pandit, Ray Kinsella, Neil Horman, Wang Haiyue

Auxiliary [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary_device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
---
 MAINTAINERS                               |   5 +
 drivers/bus/auxiliary/auxiliary_common.c  | 408 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
 drivers/bus/auxiliary/meson.build         |  11 +
 drivers/bus/auxiliary/private.h           | 120 +++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 199 +++++++++++
 drivers/bus/auxiliary/version.map         |  10 +
 drivers/bus/meson.build                   |   1 +
 9 files changed, 959 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 44f3d322ed..ccab5b09c8 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -522,6 +522,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..cef85ae991
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,408 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+
+int auxiliary_bus_logtype;
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary Driver and Device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	/* no initialization when marked as blocked, return without error */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+		return -EINVAL;
+	}
+
+	dev->driver = dr;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+		      dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
+		      dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	const char **out = addr;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (dr->match(name))
+			break;
+	}
+	if (dr != NULL && addr != NULL)
+		*out = name;
+	return dr != NULL ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map)
+		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap)
+		return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..5a1b029839
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..b75bb4d4a6
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/**
+ * @file
+ * Linux auxiliary probing.
+ */
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?\n",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..f85608afd0
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+headers = files('rte_bus_auxiliary.h')
+sources = files('auxiliary_common.c',
+	'auxiliary_params.c')
+if is_linux
+	sources += files('linux/auxiliary.c')
+endif
+deps += ['kvargs']
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..3529348900
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+/**
+ * Test whether the auxiliary device exist
+ *
+ * @param name
+ *  Auxiliary device name
+ * @return
+ *  true on exists, false otherwise
+ */
+bool auxiliary_dev_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Setup or update device when being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param aux_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_aux_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_aux_dev
+ *	AUXILIARY device to be added before exist_aux_dev
+ * @return void
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param aux_drv
+ *      auxiliary driver
+ * @param aux_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..d681464602
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _RTE_BUS_AUXILIARY_H_
+#define _RTE_BUS_AUXILIARY_H_
+
+/**
+ * @file
+ *
+ * RTE Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(*rte_auxiliary_match_t) (const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(*rte_auxiliary_probe_t) (struct rte_auxiliary_driver *drv,
+				     struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (*rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (*rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (*rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t match;         /**< Device match function. */
+	rte_auxiliary_probe_t probe;         /**< Device Probe function. */
+	rte_auxiliary_remove_t remove;       /**< Device Remove function. */
+	rte_auxiliary_dma_map_t dma_map;     /**< Device dma map function. */
+	rte_auxiliary_dma_unmap_t dma_unmap; /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&auxiliary_drv);		\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BUS_AUXILIARY_H_ */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..3d270baea7
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,10 @@
+DPDK_21 {
+	local: *;
+};
+
+EXPERIMENTAL {
+	global:
+
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index c770c6ba1e..791536e606 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
  2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
@ 2021-05-11  9:47     ` Kinsella, Ray
  2021-06-10  3:30       ` Xueming(Steven) Li
  2021-06-08  7:53     ` Thomas Monjalon
                       ` (4 subsequent siblings)
  5 siblings, 1 reply; 28+ messages in thread
From: Kinsella, Ray @ 2021-05-11  9:47 UTC (permalink / raw)
  To: Xueming Li, NBU-Contact-Thomas Monjalon
  Cc: dev, Parav Pandit, Neil Horman, Wang Haiyue



On 10/05/2021 14:47, Xueming Li wrote:
> Auxiliary [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary_device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> Cc: Wang Haiyue <haiyue.wang@intel.com>
> ---
>  MAINTAINERS                               |   5 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 408 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
>  drivers/bus/auxiliary/meson.build         |  11 +
>  drivers/bus/auxiliary/private.h           | 120 +++++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 199 +++++++++++
>  drivers/bus/auxiliary/version.map         |  10 +
>  drivers/bus/meson.build                   |   1 +
>  9 files changed, 959 insertions(+)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 44f3d322ed..ccab5b09c8 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -522,6 +522,11 @@ F: doc/guides/mempool/octeontx2.rst
>  Bus Drivers
>  -----------
>  
> +Auxiliary bus driver
> +M: Parav Pandit <parav@nvidia.com>
> +M: Xueming Li <xuemingl@nvidia.com>
> +F: drivers/bus/auxiliary/
> +
>  Intel FPGA bus
>  M: Rosen Xu <rosen.xu@intel.com>
>  F: drivers/bus/ifpga/
> diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
> new file mode 100644
> index 0000000000..cef85ae991
> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
> @@ -0,0 +1,408 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdbool.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <rte_errno.h>
> +#include <rte_interrupts.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_eal.h>
> +#include <rte_eal_paging.h>
> +#include <rte_string_fns.h>
> +#include <rte_common.h>
> +#include <rte_devargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +
> +int auxiliary_bus_logtype;
> +
> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name)
> +{
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
> +		if (strcmp(devargs->name, name) == 0)
> +			return devargs;
> +	}
> +	return NULL;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist
> + */
> +__rte_weak bool
> +auxiliary_dev_exists(const char *name)
> +{
> +	RTE_SET_USED(name);
> +	return false;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + */
> +__rte_weak int
> +auxiliary_scan(void)
> +{
> +	return 0;
> +}
> +
> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
> +{
> +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> +}
> +
> +/*
> + * Match the auxiliary Driver and Device using driver function.
> + */
> +bool
> +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		const struct rte_auxiliary_device *aux_dev)
> +{
> +	if (aux_drv->match == NULL)
> +		return false;
> +	return aux_drv->match(aux_dev->name);
> +}
> +
> +/*
> + * Call the probe() function of the driver.
> + */
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> +			       struct rte_auxiliary_device *dev)
> +{
> +	enum rte_iova_mode iova_mode;
> +	int ret;
> +
> +	if ((dr == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	/* The device is not blocked; Check if driver supports it */
> +	if (!auxiliary_match(dr, dev))
> +		/* Match of device and driver failed */
> +		return 1;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	/* no initialization when marked as blocked, return without error */
> +	if (dev->device.devargs != NULL &&
> +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> +		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
> +		return -1;
> +	}
> +
> +	if (dev->device.numa_node < 0) {
> +		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
> +		dev->device.numa_node = 0;
> +	}
> +
> +	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
> +
> +	iova_mode = rte_eal_iova_mode();
> +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) &&
> +	    iova_mode != RTE_IOVA_VA) {
> +		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
> +		return -EINVAL;
> +	}
> +
> +	dev->driver = dr;
> +
> +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
> +		      dr->driver.name, dev->name, dev->device.numa_node);
> +	ret = dr->probe(dr, dev);
> +	if (ret)
> +		dev->driver = NULL;
> +	else
> +		dev->device.driver = &dr->driver;
> +
> +	return ret;
> +}
> +
> +/*
> + * Call the remove() function of the driver.
> + */
> +static int
> +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr;
> +	int ret = 0;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	dr = dev->driver;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
> +		      dev->name, dr->driver.name);
> +
> +	if (dr->remove) {
> +		ret = dr->remove(dev);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	/* clear driver structure */
> +	dev->driver = NULL;
> +	dev->device.driver = NULL;
> +
> +	return 0;
> +}
> +
> +/*
> + * Call the probe() function of all registered driver for the given device.
> + * Return < 0 if initialization failed.
> + * Return 1 if no driver is found for this device.
> + */
> +static int
> +auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr;
> +	int rc;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
> +		if (!dr->match(dev->name))
> +			continue;
> +
> +		rc = rte_auxiliary_probe_one_driver(dr, dev);
> +		if (rc < 0)
> +			/* negative value is an error */
> +			return rc;
> +		if (rc > 0)
> +			/* positive value means driver doesn't support it */
> +			continue;
> +		return 0;
> +	}
> +	return 1;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and call the probe() function for
> + *
> + * all registered drivers that have a matching entry in its id_table
> + * for discovered devices.
> + */
> +static int
> +auxiliary_probe(void)
> +{
> +	struct rte_auxiliary_device *dev = NULL;
> +	size_t probed = 0, failed = 0;
> +	int ret = 0;
> +
> +	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
> +		probed++;
> +
> +		ret = auxiliary_probe_all_drivers(dev);
> +		if (ret < 0) {
> +			if (ret != -EEXIST) {
> +				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
> +					      dev->name);
> +				rte_errno = errno;
> +				failed++;
> +			}
> +			ret = 0;
> +		}
> +	}
> +
> +	return (probed && probed == failed) ? -1 : 0;
> +}
> +
> +static int
> +auxiliary_parse(const char *name, void *addr)
> +{
> +	struct rte_auxiliary_driver *dr = NULL;
> +	const char **out = addr;
> +
> +	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
> +		if (dr->match(name))
> +			break;
> +	}
> +	if (dr != NULL && addr != NULL)
> +		*out = name;
> +	return dr != NULL ? 0 : -1;
> +}
> +
> +/* register a driver */
> +void
> +rte_auxiliary_register(struct rte_auxiliary_driver *driver)
> +{
> +	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
> +	driver->bus = &auxiliary_bus;
> +}
> +
> +/* unregister a driver */
> +void
> +rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
> +{
> +	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
> +	driver->bus = NULL;
> +}
> +
> +/* Add a device to auxiliary bus */
> +void
> +auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
> +{
> +	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
> +}
> +
> +/* Insert a device into a predefined position in auxiliary bus */
> +void
> +auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			struct rte_auxiliary_device *new_aux_dev)
> +{
> +	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
> +}
> +
> +/* Remove a device from auxiliary bus */
> +static void
> +rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
> +{
> +	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
> +}
> +
> +static struct rte_device *
> +auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> +		      const void *data)
> +{
> +	const struct rte_auxiliary_device *pstart;
> +	struct rte_auxiliary_device *adev;
> +
> +	if (start != NULL) {
> +		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
> +		adev = TAILQ_NEXT(pstart, next);
> +	} else {
> +		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
> +	}
> +	while (adev != NULL) {
> +		if (cmp(&adev->device, data) == 0)
> +			return &adev->device;
> +		adev = TAILQ_NEXT(adev, next);
> +	}
> +	return NULL;
> +}
> +
> +static int
> +auxiliary_plug(struct rte_device *dev)
> +{
> +	if (!auxiliary_dev_exists(dev->name))
> +		return -ENOENT;
> +	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
> +}
> +
> +static int
> +auxiliary_unplug(struct rte_device *dev)
> +{
> +	struct rte_auxiliary_device *adev;
> +	int ret;
> +
> +	adev = RTE_DEV_TO_AUXILIARY(dev);
> +	ret = rte_auxiliary_driver_remove_dev(adev);
> +	if (ret == 0) {
> +		rte_auxiliary_remove_device(adev);
> +		rte_devargs_remove(dev->devargs);
> +		free(adev);
> +	}
> +	return ret;
> +}
> +
> +static int
> +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> +{
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || !aux_dev->driver) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_map)
> +		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;
> +}
> +
> +static int
> +auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
> +		    size_t len)
> +{
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || !aux_dev->driver) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_unmap)
> +		return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;
> +}
> +
> +bool
> +auxiliary_ignore_device(const char *name)
> +{
> +	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
> +
> +	switch (auxiliary_bus.bus.conf.scan_mode) {
> +	case RTE_BUS_SCAN_ALLOWLIST:
> +		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
> +			return false;
> +		break;
> +	case RTE_BUS_SCAN_UNDEFINED:
> +	case RTE_BUS_SCAN_BLOCKLIST:
> +		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
> +			return false;
> +		break;
> +	}
> +	return true;
> +}
> +
> +static enum rte_iova_mode
> +auxiliary_get_iommu_class(void)
> +{
> +	const struct rte_auxiliary_driver *drv;
> +
> +	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
> +		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
> +			return RTE_IOVA_VA;
> +	}
> +
> +	return RTE_IOVA_DC;
> +}
> +
> +struct rte_auxiliary_bus auxiliary_bus = {
> +	.bus = {
> +		.scan = auxiliary_scan,
> +		.probe = auxiliary_probe,
> +		.find_device = auxiliary_find_device,
> +		.plug = auxiliary_plug,
> +		.unplug = auxiliary_unplug,
> +		.parse = auxiliary_parse,
> +		.dma_map = auxiliary_dma_map,
> +		.dma_unmap = auxiliary_dma_unmap,
> +		.get_iommu_class = auxiliary_get_iommu_class,
> +		.dev_iterate = auxiliary_dev_iterate,
> +	},
> +	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
> +	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
> +};
> +
> +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> +RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);
> diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
> new file mode 100644
> index 0000000000..5a1b029839
> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_params.c
> @@ -0,0 +1,58 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +
> +#include <rte_bus.h>
> +#include <rte_dev.h>
> +#include <rte_errno.h>
> +#include <rte_kvargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +enum auxiliary_params {
> +	RTE_AUXILIARY_PARAM_NAME,
> +};
> +
> +static const char * const auxiliary_params_keys[] = {
> +	[RTE_AUXILIARY_PARAM_NAME] = "name",
> +};
> +
> +static int
> +auxiliary_dev_match(const struct rte_device *dev,
> +	      const void *_kvlist)
> +{
> +	const struct rte_kvargs *kvlist = _kvlist;
> +	int ret;
> +
> +	ret = rte_kvargs_process(kvlist,
> +			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
> +			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
> +
> +	return ret != 0 ? -1 : 0;
> +}
> +
> +void *
> +auxiliary_dev_iterate(const void *start,
> +		    const char *str,
> +		    const struct rte_dev_iterator *it __rte_unused)
> +{
> +	rte_bus_find_device_t find_device;
> +	struct rte_kvargs *kvargs = NULL;
> +	struct rte_device *dev;
> +
> +	if (str != NULL) {
> +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> +		if (kvargs == NULL) {
> +			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
> +			rte_errno = EINVAL;
> +			return NULL;
> +		}
> +	}
> +	find_device = auxiliary_bus.bus.find_device;
> +	dev = find_device(start, auxiliary_dev_match, kvargs);
> +	rte_kvargs_free(kvargs);
> +	return dev;
> +}
> diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
> new file mode 100644
> index 0000000000..b75bb4d4a6
> --- /dev/null
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> @@ -0,0 +1,147 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +#include <dirent.h>
> +
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <eal_filesystem.h>
> +
> +#include "../rte_bus_auxiliary.h"
> +#include "../private.h"
> +
> +#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
> +
> +/**
> + * @file
> + * Linux auxiliary probing.
> + */
> +
> +/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
> +static int
> +auxiliary_scan_one(const char *dirname, const char *name)
> +{
> +	struct rte_auxiliary_device *dev;
> +	struct rte_auxiliary_device *dev2;
> +	char filename[PATH_MAX];
> +	unsigned long tmp;
> +	int ret;
> +
> +	dev = malloc(sizeof(*dev));
> +	if (dev == NULL)
> +		return -1;
> +
> +	memset(dev, 0, sizeof(*dev));
> +	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
> +		free(dev);
> +		return -1;
> +	}
> +	dev->device.name = dev->name;
> +	dev->device.bus = &auxiliary_bus.bus;
> +
> +	/* Get numa node, default to 0 if not present */
> +	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
> +		 dirname, name);
> +	if (access(filename, F_OK) != -1) {
> +		if (eal_parse_sysfs_value(filename, &tmp) == 0)
> +			dev->device.numa_node = tmp;
> +		else
> +			dev->device.numa_node = -1;
> +	} else {
> +		dev->device.numa_node = 0;
> +	}
> +
> +	auxiliary_on_scan(dev);
> +
> +	/* Device is valid, add in list (sorted) */
> +	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
> +		ret = strcmp(dev->name, dev2->name);
> +		if (ret > 0)
> +			continue;
> +		if (ret < 0) {
> +			auxiliary_insert_device(dev2, dev);
> +		} else { /* already registered */
> +			if (rte_dev_is_probed(&dev2->device) &&
> +			    dev2->device.devargs != dev->device.devargs) {
> +				/* To probe device with new devargs. */
> +				rte_devargs_remove(dev2->device.devargs);
> +				auxiliary_on_scan(dev2);
> +			}
> +			free(dev);
> +		}
> +		return 0;
> +	}
> +	auxiliary_add_device(dev);
> +	return 0;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist
> + */
> +bool
> +auxiliary_dev_exists(const char *name)
> +{
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +
> +	snprintf(dirname, sizeof(dirname), "%s/%s",
> +		 AUXILIARY_SYSFS_PATH, name);
> +	dir = opendir(dirname);
> +	if (dir == NULL)
> +		return false;
> +	closedir(dir);
> +	return true;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + */
> +int
> +auxiliary_scan(void)
> +{
> +	struct dirent *e;
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +	struct rte_auxiliary_driver *drv;
> +
> +	dir = opendir(AUXILIARY_SYSFS_PATH);
> +	if (dir == NULL) {
> +		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?\n",
> +			      AUXILIARY_SYSFS_PATH);
> +		return 0;
> +	}
> +
> +	while ((e = readdir(dir)) != NULL) {
> +		if (e->d_name[0] == '.')
> +			continue;
> +
> +		if (auxiliary_ignore_device(e->d_name))
> +			continue;
> +
> +		snprintf(dirname, sizeof(dirname), "%s/%s",
> +			 AUXILIARY_SYSFS_PATH, e->d_name);
> +
> +		/* Ignore if no driver can handle. */
> +		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
> +			if (drv->match(e->d_name))
> +				break;
> +		}
> +		if (drv == NULL)
> +			continue;
> +
> +		if (auxiliary_scan_one(dirname, e->d_name) < 0)
> +			goto error;
> +	}
> +	closedir(dir);
> +	return 0;
> +
> +error:
> +	closedir(dir);
> +	return -1;
> +}
> diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
> new file mode 100644
> index 0000000000..f85608afd0
> --- /dev/null
> +++ b/drivers/bus/auxiliary/meson.build
> @@ -0,0 +1,11 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2021 Mellanox Technologies, Ltd
> +
> +headers = files('rte_bus_auxiliary.h')
> +sources = files('auxiliary_common.c',
> +	'auxiliary_params.c')
> +if is_linux
> +	sources += files('linux/auxiliary.c')
> +endif
> +deps += ['kvargs']
> +
> diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
> new file mode 100644
> index 0000000000..3529348900
> --- /dev/null
> +++ b/drivers/bus/auxiliary/private.h
> @@ -0,0 +1,120 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef _AUXILIARY_PRIVATE_H_
> +#define _AUXILIARY_PRIVATE_H_
> +
> +#include <stdbool.h>
> +#include <stdio.h>
> +#include "rte_bus_auxiliary.h"
> +
> +extern struct rte_auxiliary_bus auxiliary_bus;
> +extern int auxiliary_bus_logtype;
> +
> +#define AUXILIARY_LOG(level, fmt, args...) \
> +	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
> +		__func__, ##args)
> +
> +/* Auxiliary bus iterators */
> +#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
> +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> +
> +#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
> +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> +
> +/**
> + * Test whether the auxiliary device exist
> + *
> + * @param name
> + *  Auxiliary device name
> + * @return
> + *  true on exists, false otherwise
> + */
> +bool auxiliary_dev_exists(const char *name);
> +
> +/**
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + *
> + * @return
> + *  0 on success, negative on error
> + */
> +int auxiliary_scan(void);
> +
> +/**
> + * Setup or update device when being scanned.
> + *
> + * @param aux_dev
> + *	AUXILIARY device.
> + */
> +void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
> +
> +/**
> + * Validate whether a device with given auxiliary device should be ignored
> + * or not.
> + *
> + * @param name
> + *	Auxiliary name of device to be validated
> + * @return
> + *	true: if device is to be ignored,
> + *	false: if device is to be scanned,
> + */
> +bool auxiliary_ignore_device(const char *name);
> +
> +/**
> + * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
> + * list). This function also updates the bus references of the auxiliary
> + * Device (and the generic device object embedded within.
> + *
> + * @param aux_dev
> + *	AUXILIARY device to add
> + * @return void
> + */
> +void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
> +
> +/**
> + * Insert an auxiliary device in the auxiliary bus at a particular location
> + * in the device list. It also updates the auxiliary bus reference of the
> + * new devices to be inserted.
> + *
> + * @param exist_aux_dev
> + *	Existing auxiliary device in auxiliary bus
> + * @param new_aux_dev
> + *	AUXILIARY device to be added before exist_aux_dev
> + * @return void
> + */
> +void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			     struct rte_auxiliary_device *new_aux_dev);
> +
> +/**
> + * Match the auxiliary Driver and Device by driver function
> + *
> + * @param aux_drv
> + *      auxiliary driver
> + * @param aux_dev
> + *      auxiliary device to match against the driver
> + * @return
> + *      the driver can handle the device
> + */
> +bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		     const struct rte_auxiliary_device *aux_dev);
> +
> +/**
> + * Iterate over internal devices, matching any device against the provided
> + * string.
> + *
> + * @param start
> + *   Iteration starting point.
> + * @param str
> + *   Device string to match against.
> + * @param it
> + *   (unused) iterator structure.
> + * @return
> + *   A pointer to the next matching device if any.
> + *   NULL otherwise.
> + */
> +void *auxiliary_dev_iterate(const void *start, const char *str,
> +			    const struct rte_dev_iterator *it);
> +
> +#endif /* _AUXILIARY_PRIVATE_H_ */
> diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> new file mode 100644
> index 0000000000..d681464602
> --- /dev/null
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> @@ -0,0 +1,199 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef _RTE_BUS_AUXILIARY_H_
> +#define _RTE_BUS_AUXILIARY_H_
> +
> +/**
> + * @file
> + *
> + * RTE Auxiliary Bus Interface.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <sys/queue.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +
> +#include <rte_debug.h>
> +#include <rte_interrupts.h>
> +#include <rte_dev.h>
> +#include <rte_bus.h>
> +#include <rte_kvargs.h>
> +
> +/* Forward declarations */
> +struct rte_auxiliary_driver;
> +struct rte_auxiliary_bus;
> +struct rte_auxiliary_device;
> +
> +/**
> + * Match function for the driver to decide if device can be handled.
> + *
> + * @param name
> + *   Pointer to the auxiliary device name.
> + * @return
> + *   Whether the driver can handle the auxiliary device.
> + */
> +typedef bool(*rte_auxiliary_match_t) (const char *name);
> +
> +/**
> + * Initialization function for the driver called during auxiliary probing.
> + *
> + * @param drv
> + *   Pointer to the auxiliary driver.
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int(*rte_auxiliary_probe_t) (struct rte_auxiliary_driver *drv,
> +				     struct rte_auxiliary_device *dev);
> +
> +/**
> + * Uninitialization function for the driver called during hotplugging.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (*rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
> +
> +/**
> + * Driver-specific DMA mapping. After a successful call the device
> + * will be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be mapped.
> + * @param iova
> + *   Starting IOVA address of memory to be mapped.
> + * @param len
> + *   Length of memory segment being mapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (*rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
> +				       void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * Driver-specific DMA un-mapping. After a successful call the device
> + * will not be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be unmapped.
> + * @param iova
> + *   Starting IOVA address of memory to be unmapped.
> + * @param len
> + *   Length of memory segment being unmapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (*rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> +					 void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * A structure describing an auxiliary device.
> + */
> +struct rte_auxiliary_device {
> +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> +	struct rte_device device;                 /**< Inherit core device */
> +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> +};
> +
> +/** List of auxiliary devices */
> +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> +/** List of auxiliary drivers */
> +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> +
> +/**
> + * Structure describing the auxiliary bus
> + */
> +struct rte_auxiliary_bus {
> +	struct rte_bus bus;                  /**< Inherit the generic class */
> +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
> +};
> +
> +/**
> + * A structure describing an auxiliary driver.
> + */
> +struct rte_auxiliary_driver {
> +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> +	struct rte_driver driver;            /**< Inherit core driver. */
> +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> +	rte_auxiliary_match_t match;         /**< Device match function. */
> +	rte_auxiliary_probe_t probe;         /**< Device Probe function. */
> +	rte_auxiliary_remove_t remove;       /**< Device Remove function. */
> +	rte_auxiliary_dma_map_t dma_map;     /**< Device dma map function. */
> +	rte_auxiliary_dma_unmap_t dma_unmap; /**< Device dma unmap function. */
> +	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
> +};
> +
> +/**
> + * @internal
> + * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
> + */
> +#define RTE_DEV_TO_AUXILIARY(ptr) \
> +	container_of(ptr, struct rte_auxiliary_device, device)
> +
> +#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
> +	container_of(ptr, const struct rte_auxiliary_device, device)
> +
> +#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
> +	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
> +
> +/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
> +#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
> +
> +/**
> + * Register an auxiliary driver.
> + *
> + * @param driver
> + *   A pointer to a rte_auxiliary_driver structure describing the driver
> + *   to be registered.
> + */
> +__rte_experimental
> +void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
> +
> +/** Helper for auxiliary device registration from driver instance */
> +#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
> +	RTE_INIT(auxiliaryinitfn_##nm)				\
> +	{							\
> +		(auxiliary_drv).driver.name = RTE_STR(nm);	\
> +		rte_auxiliary_register(&auxiliary_drv);		\
> +	}							\
> +	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
> +
> +/**
> + * Unregister an auxiliary driver.
> + *
> + * @param driver
> + *   A pointer to a rte_auxiliary_driver structure describing the driver
> + *   to be unregistered.
> + */
> +__rte_experimental
> +void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* _RTE_BUS_AUXILIARY_H_ */
> diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
> new file mode 100644
> index 0000000000..3d270baea7
> --- /dev/null
> +++ b/drivers/bus/auxiliary/version.map
> @@ -0,0 +1,10 @@
> +DPDK_21 {
> +	local: *;
> +};
> +
> +EXPERIMENTAL {
> +	global:
> +

Add a comment here to indicate the release in which the expiermental symbols where added. 

> +	rte_auxiliary_register;
> +	rte_auxiliary_unregister;
> +};
> diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
> index c770c6ba1e..791536e606 100644
> --- a/drivers/bus/meson.build
> +++ b/drivers/bus/meson.build
> @@ -2,6 +2,7 @@
>  # Copyright(c) 2017 Intel Corporation
>  
>  drivers = [
> +        'auxiliary',
>          'dpaa',
>          'fslmc',
>          'ifpga',
> 

^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
  2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
  2021-05-11  9:47     ` Kinsella, Ray
@ 2021-06-08  7:53     ` Thomas Monjalon
  2021-06-08  8:41       ` Wang, Haiyue
  2021-06-10  6:30       ` Xueming(Steven) Li
  2021-06-13  8:19     ` [dpdk-dev] [PATCH v3 1/2] devargs: add common key definition Xueming Li
                       ` (3 subsequent siblings)
  5 siblings, 2 replies; 28+ messages in thread
From: Thomas Monjalon @ 2021-06-08  7:53 UTC (permalink / raw)
  To: Xueming Li; +Cc: dev, Parav Pandit, Ray Kinsella, Wang Haiyue, andrew.rybchenko

10/05/2021 15:47, Xueming Li:
> Auxiliary [1] provides a way to split function into child-devices

Auxiliary -> Auxiliary bus

> representing sub-domains of functionality. Each auxiliary_device

auxiliary_device -> auxiliary device

> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
[...]
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
[...]
> +int auxiliary_bus_logtype;

You don't need to declare this variable, it is done in a macro.

> +
> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name)
> +{
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {

"auxiliary" should be defined as a macro.

[...]
> +/*
> + * Test whether the auxiliary device exist
> + */
> +__rte_weak bool

The comment should explain it is a stub for OS
not supporting auxiliary bus.

> +auxiliary_dev_exists(const char *name)
> +{
> +	RTE_SET_USED(name);
> +	return false;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + */

Same here about the stub comment.

> +__rte_weak int
> +auxiliary_scan(void)
> +{
> +	return 0;
> +}
> +

Please insert a comment here.

> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
> +{
> +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> +}
> +
> +/*
> + * Match the auxiliary Driver and Device using driver function.

driver and device do not need capital letters

> + */
> +bool
> +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		const struct rte_auxiliary_device *aux_dev)
> +{
> +	if (aux_drv->match == NULL)
> +		return false;
> +	return aux_drv->match(aux_dev->name);
> +}
> +
> +/*
> + * Call the probe() function of the driver.
> + */
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> +			       struct rte_auxiliary_device *dev)
> +{
> +	enum rte_iova_mode iova_mode;
> +	int ret;
> +
> +	if ((dr == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	/* The device is not blocked; Check if driver supports it */

Please keep all the comments more uniform by starting with a capital
and ends with a point.

[...]
> +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> +RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);

Please replace with RTE_LOG_REGISTER_DEFAULT.

[...]
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
[...]
> +/**
> + * @file
> + * Linux auxiliary probing.
> + */

No need of doxygen comment in a .c file.

[...]
> +++ b/drivers/bus/auxiliary/meson.build
> @@ -0,0 +1,11 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2021 Mellanox Technologies, Ltd
> +
> +headers = files('rte_bus_auxiliary.h')
> +sources = files('auxiliary_common.c',
> +	'auxiliary_params.c')
> +if is_linux
> +	sources += files('linux/auxiliary.c')
> +endif
> +deps += ['kvargs']

Please change the indent with spaces
and check with devtools/check-meson.py

[...]
> +++ b/drivers/bus/auxiliary/private.h
[...]
> +/* Auxiliary bus iterators */
> +#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\

Please avoid tabs at the end of the line.
A space is enough before the backslash.

> +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> +
> +#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
> +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> +
> +/**
> + * Test whether the auxiliary device exist
> + *
> + * @param name
> + *  Auxiliary device name
> + * @return
> + *  true on exists, false otherwise
> + */
> +bool auxiliary_dev_exists(const char *name);

Given it is not an API, no need of doxygen.
And the function is so simple that no need of comment at all.

[...]
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> +#ifndef _RTE_BUS_AUXILIARY_H_
> +#define _RTE_BUS_AUXILIARY_H_

No need of underscores before and after.
(yes I know a lot of DPDK files use such scheme)

> +
> +/**
> + * @file
> + *
> + * RTE Auxiliary Bus Interface.

No need of RTE, it has no meaning here.

> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <sys/queue.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +
> +#include <rte_debug.h>
> +#include <rte_interrupts.h>
> +#include <rte_dev.h>
> +#include <rte_bus.h>
> +#include <rte_kvargs.h>
> +
> +/* Forward declarations */
> +struct rte_auxiliary_driver;
> +struct rte_auxiliary_bus;
> +struct rte_auxiliary_device;
> +
> +/**
> + * Match function for the driver to decide if device can be handled.
> + *
> + * @param name
> + *   Pointer to the auxiliary device name.
> + * @return
> + *   Whether the driver can handle the auxiliary device.
> + */
> +typedef bool(*rte_auxiliary_match_t) (const char *name);

I disagree with the earlier comment asking for typedef pointer
(based on one of my patches).
Actually Andrew's suggestion makes sense:
	http://mails.dpdk.org/archives/dev/2021-June/210665.html

[...]
> +++ b/drivers/bus/auxiliary/version.map
> +DPDK_21 {
> +	local: *;
> +};

We don't need this empty section.

> +
> +EXPERIMENTAL {
> +	global:
> +

As asked by Ray, we should add this line:
	# added in 21.08

> +	rte_auxiliary_register;
> +	rte_auxiliary_unregister;
> +};

Release notes are missing.



^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
  2021-06-08  7:53     ` Thomas Monjalon
@ 2021-06-08  8:41       ` Wang, Haiyue
  2021-06-10  6:29         ` Xueming(Steven) Li
  2021-06-10  6:30       ` Xueming(Steven) Li
  1 sibling, 1 reply; 28+ messages in thread
From: Wang, Haiyue @ 2021-06-08  8:41 UTC (permalink / raw)
  To: Thomas Monjalon, Xueming Li
  Cc: dev, Parav Pandit, Ray Kinsella, andrew.rybchenko

Hi Andrew,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, June 8, 2021 15:53
> To: Xueming Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Wang, Haiyue
> <haiyue.wang@intel.com>; andrew.rybchenko@oktetlabs.ru
> Subject: Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> 


> > +
> > +/**
> > + * Match function for the driver to decide if device can be handled.
> > + *
> > + * @param name
> > + *   Pointer to the auxiliary device name.
> > + * @return
> > + *   Whether the driver can handle the auxiliary device.
> > + */
> > +typedef bool(*rte_auxiliary_match_t) (const char *name);
> 
> I disagree with the earlier comment asking for typedef pointer
> (based on one of my patches).
> Actually Andrew's suggestion makes sense:
> 	http://mails.dpdk.org/archives/dev/2021-June/210665.html
> 

I didn't get the error, but the same warning, I missed something ? ;-)

1. w/o pointer

#include <stdio.h>

typedef int (gpu_malloc_t)(void *dev, size_t size, void **ptr);

static gpu_malloc_t *mlx5_gpu_malloc_fn;

static int
mlx5_gpu_malloc(size_t size, void **ptr)
{
	return 0;
}

int main()
{
	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;

	mlx5_gpu_malloc_fn(NULL, 0, NULL);

	return 0;
}


gcc -Wall fun.c
fun.c: In function 'main':
fun.c:15:21: warning: assignment to 'int (*)(void *, size_t,  void **)' {aka 'int (*)(void *, long unsigned int,  void **)'} from incompatible pointer type 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-types]
   15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
      |                     ^



2. w pointer

#include <stdio.h>

typedef int (*gpu_malloc_t)(void *dev, size_t size, void **ptr);

static gpu_malloc_t mlx5_gpu_malloc_fn;

static int
mlx5_gpu_malloc(size_t size, void **ptr)
{
	return 0;
}

int main()
{
	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;

	mlx5_gpu_malloc_fn(NULL, 0, NULL);

	return 0;
}

gcc -Wall fun.c
fun.c: In function 'main':
fun.c:15:21: warning: assignment to 'gpu_malloc_t' {aka 'int (*)(void *, long unsigned int,  void **)'} from incompatible pointer type 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-types]
   15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
      |                     ^

> 


^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
  2021-05-11  9:47     ` Kinsella, Ray
@ 2021-06-10  3:30       ` Xueming(Steven) Li
  0 siblings, 0 replies; 28+ messages in thread
From: Xueming(Steven) Li @ 2021-06-10  3:30 UTC (permalink / raw)
  To: Kinsella, Ray, NBU-Contact-Thomas Monjalon
  Cc: dev, Parav Pandit, Neil Horman, Wang Haiyue



> -----Original Message-----
> From: Kinsella, Ray <mdr@ashroe.eu>
> Sent: Tuesday, May 11, 2021 5:48 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Neil Horman <nhorman@tuxdriver.com>; Wang Haiyue
> <haiyue.wang@intel.com>
> Subject: Re: [RFC v2] bus/auxiliary: introduce auxiliary bus
> 


// snip //

> > diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
> > new file mode 100644
> > index 0000000000..3d270baea7
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/version.map
> > @@ -0,0 +1,10 @@
> > +DPDK_21 {
> > +	local: *;
> > +};
> > +
> > +EXPERIMENTAL {
> > +	global:
> > +
> 
> Add a comment here to indicate the release in which the expiermental symbols where added.

Good suggestion, thanks!

> 
> > +	rte_auxiliary_register;
> > +	rte_auxiliary_unregister;
> > +};

// snip //

^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
  2021-06-08  8:41       ` Wang, Haiyue
@ 2021-06-10  6:29         ` Xueming(Steven) Li
  2021-06-10 15:16           ` Wang, Haiyue
  0 siblings, 1 reply; 28+ messages in thread
From: Xueming(Steven) Li @ 2021-06-10  6:29 UTC (permalink / raw)
  To: Wang, Haiyue, NBU-Contact-Thomas Monjalon
  Cc: dev, Parav Pandit, Ray Kinsella, andrew.rybchenko



> -----Original Message-----
> From: Wang, Haiyue <haiyue.wang@intel.com>
> Sent: Tuesday, June 8, 2021 4:41 PM
> To: NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; andrew.rybchenko@oktetlabs.ru
> Subject: RE: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> 
> Hi Andrew,
> 
> > -----Original Message-----
> > From: Thomas Monjalon <thomas@monjalon.net>
> > Sent: Tuesday, June 8, 2021 15:53
> > To: Xueming Li <xuemingl@nvidia.com>
> > Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> > <mdr@ashroe.eu>; Wang, Haiyue <haiyue.wang@intel.com>;
> > andrew.rybchenko@oktetlabs.ru
> > Subject: Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary
> > bus
> >
> 
> 
> > > +
> > > +/**
> > > + * Match function for the driver to decide if device can be handled.
> > > + *
> > > + * @param name
> > > + *   Pointer to the auxiliary device name.
> > > + * @return
> > > + *   Whether the driver can handle the auxiliary device.
> > > + */
> > > +typedef bool(*rte_auxiliary_match_t) (const char *name);
> >
> > I disagree with the earlier comment asking for typedef pointer (based
> > on one of my patches).
> > Actually Andrew's suggestion makes sense:
> > 	http://mails.dpdk.org/archives/dev/2021-June/210665.html
> >
> 
> I didn't get the error, but the same warning, I missed something ? ;-)
> 
> 1. w/o pointer
> 
> #include <stdio.h>
> 
> typedef int (gpu_malloc_t)(void *dev, size_t size, void **ptr);
> 
> static gpu_malloc_t *mlx5_gpu_malloc_fn;
> 
> static int
> mlx5_gpu_malloc(size_t size, void **ptr) {
> 	return 0;
> }
> 
> int main()
> {
> 	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> 
> 	mlx5_gpu_malloc_fn(NULL, 0, NULL);
> 
> 	return 0;
> }
> 
> 
> gcc -Wall fun.c
> fun.c: In function 'main':
> fun.c:15:21: warning: assignment to 'int (*)(void *, size_t,  void **)' {aka 'int (*)(void *, long unsigned int,  void **)'} from incompatible
> pointer type 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-types]
>    15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
>       |                     ^
> 
> 
> 
> 2. w pointer
> 
> #include <stdio.h>
> 
> typedef int (*gpu_malloc_t)(void *dev, size_t size, void **ptr);
> 
> static gpu_malloc_t mlx5_gpu_malloc_fn;
> 
> static int
> mlx5_gpu_malloc(size_t size, void **ptr) {
> 	return 0;
> }
> 
> int main()
> {
> 	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> 
> 	mlx5_gpu_malloc_fn(NULL, 0, NULL);
> 
> 	return 0;
> }
> 
> gcc -Wall fun.c
> fun.c: In function 'main':
> fun.c:15:21: warning: assignment to 'gpu_malloc_t' {aka 'int (*)(void *, long unsigned int,  void **)'} from incompatible pointer type
> 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-types]
>    15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
>       |                     ^
> 
> >

I don't mind as both suggestion works.
From usage perspective, it's straightforward for anybody to see an pointer field and know "that's a pointer to a something".



^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
  2021-06-08  7:53     ` Thomas Monjalon
  2021-06-08  8:41       ` Wang, Haiyue
@ 2021-06-10  6:30       ` Xueming(Steven) Li
  1 sibling, 0 replies; 28+ messages in thread
From: Xueming(Steven) Li @ 2021-06-10  6:30 UTC (permalink / raw)
  To: NBU-Contact-Thomas Monjalon
  Cc: dev, Parav Pandit, Ray Kinsella, Wang Haiyue, andrew.rybchenko


> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, June 8, 2021 3:53 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Wang Haiyue <haiyue.wang@intel.com>;
> andrew.rybchenko@oktetlabs.ru
> Subject: Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> 
> 10/05/2021 15:47, Xueming Li:
> > Auxiliary [1] provides a way to split function into child-devices
> 
> Auxiliary -> Auxiliary bus
> 
> > representing sub-domains of functionality. Each auxiliary_device
> 
> auxiliary_device -> auxiliary device
> 
> > represents a part of its parent functionality.
> >
> > Auxiliary device is identified by unique device name, sysfs path:
> >   /sys/bus/auxiliary/devices/<name>
> >
> > [1] kernel auxiliary bus document:
> > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> [...]
> > +++ b/drivers/bus/auxiliary/auxiliary_common.c
> [...]
> > +int auxiliary_bus_logtype;
> 
> You don't need to declare this variable, it is done in a macro.
> 
> > +
> > +static struct rte_devargs *
> > +auxiliary_devargs_lookup(const char *name) {
> > +	struct rte_devargs *devargs;
> > +
> > +	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
> 
> "auxiliary" should be defined as a macro.
> 
> [...]
> > +/*
> > + * Test whether the auxiliary device exist  */ __rte_weak bool
> 
> The comment should explain it is a stub for OS not supporting auxiliary bus.
> 
> > +auxiliary_dev_exists(const char *name) {
> > +	RTE_SET_USED(name);
> > +	return false;
> > +}
> > +
> > +/*
> > + * Scan the content of the auxiliary bus, and the devices in the
> > +devices
> > + * list
> > + */
> 
> Same here about the stub comment.
> 
> > +__rte_weak int
> > +auxiliary_scan(void)
> > +{
> > +	return 0;
> > +}
> > +
> 
> Please insert a comment here.
> 
> > +void
> > +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev) {
> > +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> > +}
> > +
> > +/*
> > + * Match the auxiliary Driver and Device using driver function.
> 
> driver and device do not need capital letters
> 
> > + */
> > +bool
> > +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> > +		const struct rte_auxiliary_device *aux_dev) {
> > +	if (aux_drv->match == NULL)
> > +		return false;
> > +	return aux_drv->match(aux_dev->name); }
> > +
> > +/*
> > + * Call the probe() function of the driver.
> > + */
> > +static int
> > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > +			       struct rte_auxiliary_device *dev)
> > +{
> > +	enum rte_iova_mode iova_mode;
> > +	int ret;
> > +
> > +	if ((dr == NULL) || (dev == NULL))
> > +		return -EINVAL;
> > +
> > +	/* The device is not blocked; Check if driver supports it */
> 
> Please keep all the comments more uniform by starting with a capital
> and ends with a point.
> 
> [...]
> > +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> > +RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);
> 
> Please replace with RTE_LOG_REGISTER_DEFAULT.
> 
> [...]
> > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> [...]
> > +/**
> > + * @file
> > + * Linux auxiliary probing.
> > + */
> 
> No need of doxygen comment in a .c file.
> 
> [...]
> > +++ b/drivers/bus/auxiliary/meson.build
> > @@ -0,0 +1,11 @@
> > +# SPDX-License-Identifier: BSD-3-Clause
> > +# Copyright 2021 Mellanox Technologies, Ltd
> > +
> > +headers = files('rte_bus_auxiliary.h')
> > +sources = files('auxiliary_common.c',
> > +	'auxiliary_params.c')
> > +if is_linux
> > +	sources += files('linux/auxiliary.c')
> > +endif
> > +deps += ['kvargs']
> 
> Please change the indent with spaces
> and check with devtools/check-meson.py
> 
> [...]
> > +++ b/drivers/bus/auxiliary/private.h
> [...]
> > +/* Auxiliary bus iterators */
> > +#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
> 
> Please avoid tabs at the end of the line.
> A space is enough before the backslash.
> 
> > +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> > +
> > +#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
> > +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> > +
> > +/**
> > + * Test whether the auxiliary device exist
> > + *
> > + * @param name
> > + *  Auxiliary device name
> > + * @return
> > + *  true on exists, false otherwise
> > + */
> > +bool auxiliary_dev_exists(const char *name);
> 
> Given it is not an API, no need of doxygen.
> And the function is so simple that no need of comment at all.
> 
> [...]
> > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > +#ifndef _RTE_BUS_AUXILIARY_H_
> > +#define _RTE_BUS_AUXILIARY_H_
> 
> No need of underscores before and after.
> (yes I know a lot of DPDK files use such scheme)
> 
> > +
> > +/**
> > + * @file
> > + *
> > + * RTE Auxiliary Bus Interface.
> 
> No need of RTE, it has no meaning here.
> 
> > + */
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <limits.h>
> > +#include <errno.h>
> > +#include <sys/queue.h>
> > +#include <stdint.h>
> > +#include <inttypes.h>
> > +
> > +#include <rte_debug.h>
> > +#include <rte_interrupts.h>
> > +#include <rte_dev.h>
> > +#include <rte_bus.h>
> > +#include <rte_kvargs.h>
> > +
> > +/* Forward declarations */
> > +struct rte_auxiliary_driver;
> > +struct rte_auxiliary_bus;
> > +struct rte_auxiliary_device;
> > +
> > +/**
> > + * Match function for the driver to decide if device can be handled.
> > + *
> > + * @param name
> > + *   Pointer to the auxiliary device name.
> > + * @return
> > + *   Whether the driver can handle the auxiliary device.
> > + */
> > +typedef bool(*rte_auxiliary_match_t) (const char *name);
> 
> I disagree with the earlier comment asking for typedef pointer
> (based on one of my patches).
> Actually Andrew's suggestion makes sense:
> 	http://mails.dpdk.org/archives/dev/2021-June/210665.html
> 
> [...]
> > +++ b/drivers/bus/auxiliary/version.map
> > +DPDK_21 {
> > +	local: *;
> > +};
> 
> We don't need this empty section.
> 
> > +
> > +EXPERIMENTAL {
> > +	global:
> > +
> 
> As asked by Ray, we should add this line:
> 	# added in 21.08
> 
> > +	rte_auxiliary_register;
> > +	rte_auxiliary_unregister;
> > +};
> 
> Release notes are missing.
> 

Thanks for all the good catches, will update and reflect in next version.

^ permalink raw reply	[flat|nested] 28+ messages in thread

* Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
  2021-06-10  6:29         ` Xueming(Steven) Li
@ 2021-06-10 15:16           ` Wang, Haiyue
  0 siblings, 0 replies; 28+ messages in thread
From: Wang, Haiyue @ 2021-06-10 15:16 UTC (permalink / raw)
  To: Xueming(Steven) Li, NBU-Contact-Thomas Monjalon
  Cc: dev, Parav Pandit, Ray Kinsella, andrew.rybchenko

> -----Original Message-----
> From: Xueming(Steven) Li <xuemingl@nvidia.com>
> Sent: Thursday, June 10, 2021 14:30
> To: Wang, Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>;
> andrew.rybchenko@oktetlabs.ru
> Subject: RE: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> 
> 
> 
> > -----Original Message-----
> > From: Wang, Haiyue <haiyue.wang@intel.com>
> > Sent: Tuesday, June 8, 2021 4:41 PM
> > To: NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Xueming(Steven) Li <xuemingl@nvidia.com>
> > Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>;
> andrew.rybchenko@oktetlabs.ru
> > Subject: RE: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> >
> > Hi Andrew,
> >
> > > -----Original Message-----
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > Sent: Tuesday, June 8, 2021 15:53
> > > To: Xueming Li <xuemingl@nvidia.com>
> > > Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> > > <mdr@ashroe.eu>; Wang, Haiyue <haiyue.wang@intel.com>;
> > > andrew.rybchenko@oktetlabs.ru
> > > Subject: Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary
> > > bus
> > >
> >
> >
> > > > +
> > > > +/**
> > > > + * Match function for the driver to decide if device can be handled.
> > > > + *
> > > > + * @param name
> > > > + *   Pointer to the auxiliary device name.
> > > > + * @return
> > > > + *   Whether the driver can handle the auxiliary device.
> > > > + */
> > > > +typedef bool(*rte_auxiliary_match_t) (const char *name);
> > >
> > > I disagree with the earlier comment asking for typedef pointer (based
> > > on one of my patches).
> > > Actually Andrew's suggestion makes sense:
> > > 	http://mails.dpdk.org/archives/dev/2021-June/210665.html
> > >
> >
> > I didn't get the error, but the same warning, I missed something ? ;-)
> >
> > 1. w/o pointer
> >
> > #include <stdio.h>
> >
> > typedef int (gpu_malloc_t)(void *dev, size_t size, void **ptr);
> >
> > static gpu_malloc_t *mlx5_gpu_malloc_fn;
> >
> > static int
> > mlx5_gpu_malloc(size_t size, void **ptr) {
> > 	return 0;
> > }
> >
> > int main()
> > {
> > 	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> >
> > 	mlx5_gpu_malloc_fn(NULL, 0, NULL);
> >
> > 	return 0;
> > }
> >
> >
> > gcc -Wall fun.c
> > fun.c: In function 'main':
> > fun.c:15:21: warning: assignment to 'int (*)(void *, size_t,  void **)' {aka 'int (*)(void *, long
> unsigned int,  void **)'} from incompatible
> > pointer type 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-
> Wincompatible-pointer-types]
> >    15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> >       |                     ^
> >
> >
> >
> > 2. w pointer
> >
> > #include <stdio.h>
> >
> > typedef int (*gpu_malloc_t)(void *dev, size_t size, void **ptr);
> >
> > static gpu_malloc_t mlx5_gpu_malloc_fn;
> >
> > static int
> > mlx5_gpu_malloc(size_t size, void **ptr) {
> > 	return 0;
> > }
> >
> > int main()
> > {
> > 	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> >
> > 	mlx5_gpu_malloc_fn(NULL, 0, NULL);
> >
> > 	return 0;
> > }
> >
> > gcc -Wall fun.c
> > fun.c: In function 'main':
> > fun.c:15:21: warning: assignment to 'gpu_malloc_t' {aka 'int (*)(void *, long unsigned int,  void
> **)'} from incompatible pointer type
> > 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-
> types]
> >    15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> >       |                     ^
> >
> > >
> 
> I don't mind as both suggestion works.

I just want to understand what kind of error the compiler will complain about. ;-)

> From usage perspective, it's straightforward for anybody to see an pointer field and know "that's a
> pointer to a something".

> 


^ permalink raw reply	[flat|nested] 28+ messages in thread

* [dpdk-dev] [PATCH v3 1/2] devargs: add common key definition
  2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
  2021-05-11  9:47     ` Kinsella, Ray
  2021-06-08  7:53     ` Thomas Monjalon
@ 2021-06-13  8:19     ` Xueming Li
  2021-06-13  8:19     ` [dpdk-dev] [PATCH v3 2/2] bus/auxiliary: introduce auxiliary bus Xueming Li
                       ` (2 subsequent siblings)
  5 siblings, 0 replies; 28+ messages in thread
From: Xueming Li @ 2021-06-13  8:19 UTC (permalink / raw)
  Cc: dev, xuemingl, Matan Azrad, Shahaf Shuler, Viacheslav Ovsiienko,
	Andrew Rybchenko, Chas Williams, Min Hu (Connor),
	Beilei Xing, Jingjing Wu

Adds common devargs key definition for "bus", "class" and "driver".

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |  2 --
 drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
 drivers/common/sfc_efx/sfc_efx.h        |  2 --
 drivers/net/bonding/rte_eth_bond_args.c |  2 +-
 drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
 drivers/net/iavf/iavf_ethdev.c          |  5 ++---
 drivers/net/mlx5/mlx5.c                 |  4 ++--
 drivers/net/sfc/sfc_kvargs.c            |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
 lib/eal/common/eal_common_devargs.c     | 12 ++++++------
 lib/eal/include/rte_devargs.h           |  4 ++++
 11 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..306f2f1ab7 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -208,8 +208,6 @@ __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
 
-#define MLX5_CLASS_ARG_NAME "class"
-
 enum mlx5_class {
 	MLX5_CLASS_INVALID,
 	MLX5_CLASS_NET = RTE_BIT64(0),
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..34747c4e07 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,
 static int
 parse_class_options(const struct rte_devargs *devargs)
 {
-	const char *key = MLX5_CLASS_ARG_NAME;
+	const char *key = RTE_DEVARGS_KEY_CLASS;
 	struct rte_kvargs *kvlist;
 	int ret = 0;
 
diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
index 6b6164cb1f..c16eca60f3 100644
--- a/drivers/common/sfc_efx/sfc_efx.h
+++ b/drivers/common/sfc_efx/sfc_efx.h
@@ -19,8 +19,6 @@
 extern "C" {
 #endif
 
-#define SFC_EFX_KVARG_DEV_CLASS	"class"
-
 enum sfc_efx_dev_class {
 	SFC_EFX_DEV_CLASS_INVALID = 0,
 	SFC_EFX_DEV_CLASS_NET,
diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
index 764b1b8c8e..5406e1c934 100644
--- a/drivers/net/bonding/rte_eth_bond_args.c
+++ b/drivers/net/bonding/rte_eth_bond_args.c
@@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
 	PMD_BOND_SOCKET_ID_KVARG,
 	PMD_BOND_MAC_ADDR_KVARG,
 	PMD_BOND_AGG_MODE_KVARG,
-	"driver",
+	RTE_DEVARGS_KEY_DRIVER,
 	NULL
 };
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index cb898bdb68..1d8ca42a0f 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1660,7 +1660,6 @@ static int
 i40evf_driver_selected(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (devargs == NULL)
@@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       i40evf_check_driver_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index d688c31cfb..4e79319017 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -2440,7 +2440,6 @@ static int
 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (device_id != IAVF_DEV_ID_VF &&
@@ -2456,13 +2455,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       iavf_drv_i40evf_check_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..d0faa45944 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->max_dump_files_num = tmp;
 	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
 		config->lro.timeout = tmp;
-	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
+	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
 		DRV_LOG(DEBUG, "class argument is %s.", val);
 	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
 		config->log_hp_size = tmp;
@@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_REPRESENTOR,
 		MLX5_MAX_DUMP_FILES_NUM,
 		MLX5_LRO_TIMEOUT_USEC,
-		MLX5_CLASS_ARG_NAME,
+		RTE_DEVARGS_KEY_CLASS,
 		MLX5_HP_BUF_SIZE,
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 0efa92ed28..974c05e68e 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
 		SFC_KVARG_TX_DATAPATH,
 		SFC_KVARG_FW_VARIANT,
 		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
-		SFC_EFX_KVARG_DEV_CLASS,
+		RTE_DEVARGS_KEY_CLASS,
 		NULL,
 	};
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index e5e03e6582..8b5bfd8c3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 	unsigned long tmp;
 	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 
-	if (strcmp(key, "class") == 0)
+	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
 		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
index b31ac879a9..23aaf8b7e4 100644
--- a/lib/eal/common/eal_common_devargs.c
+++ b/lib/eal/common/eal_common_devargs.c
@@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		const char *str;
 		struct rte_kvargs *kvlist;
 	} layers[] = {
-		{ "bus=",    NULL, NULL, },
-		{ "class=",  NULL, NULL, },
-		{ "driver=", NULL, NULL, },
+		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
+		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
+		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
 	};
 	struct rte_kvargs_pair *kv = NULL;
 	struct rte_class *cls = NULL;
@@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		if (layers[i].kvlist == NULL)
 			continue;
 		kv = &layers[i].kvlist->pairs[0];
-		if (strcmp(kv->key, "bus") == 0) {
+		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
 			bus = rte_bus_find_by_name(kv->value);
 			if (bus == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
@@ -126,7 +126,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "class") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
 			cls = rte_class_find_by_name(kv->value);
 			if (cls == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
@@ -134,7 +134,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "driver") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
 			/* Ignore */
 			continue;
 		}
diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h
index 1e595b3c51..14fe7f70b6 100644
--- a/lib/eal/include/rte_devargs.h
+++ b/lib/eal/include/rte_devargs.h
@@ -25,6 +25,10 @@ extern "C" {
 #include <rte_compat.h>
 #include <rte_bus.h>
 
+#define RTE_DEVARGS_KEY_BUS "bus"
+#define RTE_DEVARGS_KEY_CLASS "class"
+#define RTE_DEVARGS_KEY_DRIVER "driver"
+
 /**
  * Type of generic device
  */
-- 
2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* [dpdk-dev] [PATCH v3 2/2] bus/auxiliary: introduce auxiliary bus
  2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
                       ` (2 preceding siblings ...)
  2021-06-13  8:19     ` [dpdk-dev] [PATCH v3 1/2] devargs: add common key definition Xueming Li
@ 2021-06-13  8:19     ` Xueming Li
  2021-06-13 12:58     ` [dpdk-dev] [PATCH v4 1/2] devargs: add common key definition Xueming Li
  2021-06-13 12:58     ` [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus Xueming Li
  5 siblings, 0 replies; 28+ messages in thread
From: Xueming Li @ 2021-06-13  8:19 UTC (permalink / raw)
  Cc: dev, xuemingl, Wang Haiyue, Thomas Monjalon, Kinsella Ray,
	Parav Pandit, Neil Horman

Auxiliary bus [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

Devargs syntax of auxiliary device:
  -a auxiliary:<name>[,args...]

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
Cc: Thomas Monjalon <thomas@monjalon.net>
Cc: Kinsella Ray <mdr@ashroe.eu>
---
 MAINTAINERS                               |   5 +
 doc/guides/rel_notes/release_21_08.rst    |   7 +
 drivers/bus/auxiliary/auxiliary_common.c  | 419 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 142 ++++++++
 drivers/bus/auxiliary/meson.build         |  11 +
 drivers/bus/auxiliary/private.h           | 112 ++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
 drivers/bus/auxiliary/version.map         |   7 +
 drivers/bus/meson.build                   |   1 +
 10 files changed, 963 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..eaf691ca6a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..b335064963 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,13 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added auxiliary bus support.**
+
+  * Auxiliary bus provides a way to split function into child-devices
+    representing sub-domains of functionality. Each auxiliary device
+    represents a part of its parent functionality.
+  * Devargs syntax of auxiliary device: -a auxiliary:<name>[,args...]
+
 
 Removed Items
 -------------
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..91008bdc80
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,419 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AXILIARY_NAME, devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+/**
+ * Update a device being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary driver and device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it. */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	/* No initialization when marked as blocked, return without error. */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+		return -EINVAL;
+	}
+
+	dev->driver = dr;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+		      dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
+		      dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	const char **out = addr;
+
+	/* Allow dummy name to prevent bus scan. */
+	if (strlen(name) == 0)
+		return 0;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (dr->match(name))
+			break;
+	}
+	if (dr != NULL && addr != NULL)
+		*out = name;
+	return dr != NULL ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map)
+		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap)
+		return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..5a1b029839
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..22e0e20493
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?\n",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..3d5c7b0d4a
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+headers = files('rte_bus_auxiliary.h')
+sources = files('auxiliary_common.c',
+    'auxiliary_params.c')
+if is_linux
+    sources += files('linux/auxiliary.c')
+endif
+deps += ['kvargs']
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..3146fcd5d3
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+bool auxiliary_dev_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Update a device being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param aux_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_aux_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_aux_dev
+ *	AUXILIARY device to be added before exist_aux_dev
+ * @return void
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param aux_drv
+ *      auxiliary driver
+ * @param aux_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..94f0790ba0
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_BUS_AUXILIARY_H
+#define RTE_BUS_AUXILIARY_H
+
+/**
+ * @file
+ *
+ * Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+#define RTE_BUS_AXILIARY_NAME "auxiliary"
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(rte_auxiliary_match_t) (const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(rte_auxiliary_probe_t) (struct rte_auxiliary_driver *drv,
+				    struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t *match;         /**< Device match function. */
+	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
+	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */
+	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
+	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_ ##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&(auxiliary_drv));	\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BUS_AUXILIARY_H */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..a52260657c
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,7 @@
+EXPERIMENTAL {
+	global:
+
+	# added in 21.08
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 410058de3a..45eab5233d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* [dpdk-dev] [PATCH v4 1/2] devargs: add common key definition
  2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
                       ` (3 preceding siblings ...)
  2021-06-13  8:19     ` [dpdk-dev] [PATCH v3 2/2] bus/auxiliary: introduce auxiliary bus Xueming Li
@ 2021-06-13 12:58     ` Xueming Li
  2021-06-13 12:58     ` [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus Xueming Li
  5 siblings, 0 replies; 28+ messages in thread
From: Xueming Li @ 2021-06-13 12:58 UTC (permalink / raw)
  Cc: dev, xuemingl, Matan Azrad, Shahaf Shuler, Viacheslav Ovsiienko,
	Andrew Rybchenko, Chas Williams, Min Hu (Connor),
	Beilei Xing, Jingjing Wu

Adds common devargs key definition for "bus", "class" and "driver".

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |  2 --
 drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
 drivers/common/sfc_efx/sfc_efx.c        |  7 +++----
 drivers/common/sfc_efx/sfc_efx.h        |  2 --
 drivers/net/bonding/rte_eth_bond_args.c |  2 +-
 drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
 drivers/net/iavf/iavf_ethdev.c          |  5 ++---
 drivers/net/mlx5/mlx5.c                 |  4 ++--
 drivers/net/sfc/sfc_kvargs.c            |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
 lib/eal/common/eal_common_devargs.c     | 12 ++++++------
 lib/eal/include/rte_devargs.h           |  4 ++++
 12 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..306f2f1ab7 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -208,8 +208,6 @@ __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
 
-#define MLX5_CLASS_ARG_NAME "class"
-
 enum mlx5_class {
 	MLX5_CLASS_INVALID,
 	MLX5_CLASS_NET = RTE_BIT64(0),
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..34747c4e07 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,
 static int
 parse_class_options(const struct rte_devargs *devargs)
 {
-	const char *key = MLX5_CLASS_ARG_NAME;
+	const char *key = RTE_DEVARGS_KEY_CLASS;
 	struct rte_kvargs *kvlist;
 	int ret = 0;
 
diff --git a/drivers/common/sfc_efx/sfc_efx.c b/drivers/common/sfc_efx/sfc_efx.c
index 0b78933d9f..2dc5545760 100644
--- a/drivers/common/sfc_efx/sfc_efx.c
+++ b/drivers/common/sfc_efx/sfc_efx.c
@@ -42,7 +42,6 @@ enum sfc_efx_dev_class
 sfc_efx_dev_class_get(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvargs;
-	const char *key = SFC_EFX_KVARG_DEV_CLASS;
 	enum sfc_efx_dev_class dev_class = SFC_EFX_DEV_CLASS_NET;
 
 	if (devargs == NULL)
@@ -52,9 +51,9 @@ sfc_efx_dev_class_get(struct rte_devargs *devargs)
 	if (kvargs == NULL)
 		return dev_class;
 
-	if (rte_kvargs_count(kvargs, key) != 0) {
-		rte_kvargs_process(kvargs, key, sfc_efx_kvarg_dev_class_handler,
-				   &dev_class);
+	if (rte_kvargs_count(kvargs, RTE_DEVARGS_KEY_CLASS) != 0) {
+		rte_kvargs_process(kvargs, RTE_DEVARGS_KEY_CLASS,
+				   sfc_efx_kvarg_dev_class_handler, &dev_class);
 	}
 
 	rte_kvargs_free(kvargs);
diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
index 6b6164cb1f..c16eca60f3 100644
--- a/drivers/common/sfc_efx/sfc_efx.h
+++ b/drivers/common/sfc_efx/sfc_efx.h
@@ -19,8 +19,6 @@
 extern "C" {
 #endif
 
-#define SFC_EFX_KVARG_DEV_CLASS	"class"
-
 enum sfc_efx_dev_class {
 	SFC_EFX_DEV_CLASS_INVALID = 0,
 	SFC_EFX_DEV_CLASS_NET,
diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
index 764b1b8c8e..5406e1c934 100644
--- a/drivers/net/bonding/rte_eth_bond_args.c
+++ b/drivers/net/bonding/rte_eth_bond_args.c
@@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
 	PMD_BOND_SOCKET_ID_KVARG,
 	PMD_BOND_MAC_ADDR_KVARG,
 	PMD_BOND_AGG_MODE_KVARG,
-	"driver",
+	RTE_DEVARGS_KEY_DRIVER,
 	NULL
 };
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index cb898bdb68..1d8ca42a0f 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1660,7 +1660,6 @@ static int
 i40evf_driver_selected(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (devargs == NULL)
@@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       i40evf_check_driver_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index d688c31cfb..4e79319017 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -2440,7 +2440,6 @@ static int
 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (device_id != IAVF_DEV_ID_VF &&
@@ -2456,13 +2455,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       iavf_drv_i40evf_check_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..d0faa45944 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->max_dump_files_num = tmp;
 	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
 		config->lro.timeout = tmp;
-	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
+	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
 		DRV_LOG(DEBUG, "class argument is %s.", val);
 	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
 		config->log_hp_size = tmp;
@@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_REPRESENTOR,
 		MLX5_MAX_DUMP_FILES_NUM,
 		MLX5_LRO_TIMEOUT_USEC,
-		MLX5_CLASS_ARG_NAME,
+		RTE_DEVARGS_KEY_CLASS,
 		MLX5_HP_BUF_SIZE,
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 0efa92ed28..974c05e68e 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
 		SFC_KVARG_TX_DATAPATH,
 		SFC_KVARG_FW_VARIANT,
 		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
-		SFC_EFX_KVARG_DEV_CLASS,
+		RTE_DEVARGS_KEY_CLASS,
 		NULL,
 	};
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index e5e03e6582..8b5bfd8c3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 	unsigned long tmp;
 	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 
-	if (strcmp(key, "class") == 0)
+	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
 		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
index b31ac879a9..23aaf8b7e4 100644
--- a/lib/eal/common/eal_common_devargs.c
+++ b/lib/eal/common/eal_common_devargs.c
@@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		const char *str;
 		struct rte_kvargs *kvlist;
 	} layers[] = {
-		{ "bus=",    NULL, NULL, },
-		{ "class=",  NULL, NULL, },
-		{ "driver=", NULL, NULL, },
+		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
+		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
+		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
 	};
 	struct rte_kvargs_pair *kv = NULL;
 	struct rte_class *cls = NULL;
@@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		if (layers[i].kvlist == NULL)
 			continue;
 		kv = &layers[i].kvlist->pairs[0];
-		if (strcmp(kv->key, "bus") == 0) {
+		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
 			bus = rte_bus_find_by_name(kv->value);
 			if (bus == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
@@ -126,7 +126,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "class") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
 			cls = rte_class_find_by_name(kv->value);
 			if (cls == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
@@ -134,7 +134,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "driver") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
 			/* Ignore */
 			continue;
 		}
diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h
index 1e595b3c51..14fe7f70b6 100644
--- a/lib/eal/include/rte_devargs.h
+++ b/lib/eal/include/rte_devargs.h
@@ -25,6 +25,10 @@ extern "C" {
 #include <rte_compat.h>
 #include <rte_bus.h>
 
+#define RTE_DEVARGS_KEY_BUS "bus"
+#define RTE_DEVARGS_KEY_CLASS "class"
+#define RTE_DEVARGS_KEY_DRIVER "driver"
+
 /**
  * Type of generic device
  */
-- 
2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

* [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus
  2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
                       ` (4 preceding siblings ...)
  2021-06-13 12:58     ` [dpdk-dev] [PATCH v4 1/2] devargs: add common key definition Xueming Li
@ 2021-06-13 12:58     ` Xueming Li
  5 siblings, 0 replies; 28+ messages in thread
From: Xueming Li @ 2021-06-13 12:58 UTC (permalink / raw)
  Cc: dev, xuemingl, Wang Haiyue, Thomas Monjalon, Kinsella Ray,
	Parav Pandit, Neil Horman

Auxiliary bus [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

Devargs syntax of auxiliary device:
  -a auxiliary:<name>[,args...]

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
Cc: Thomas Monjalon <thomas@monjalon.net>
Cc: Kinsella Ray <mdr@ashroe.eu>
---
 MAINTAINERS                               |   5 +
 doc/guides/rel_notes/release_21_08.rst    |   7 +
 drivers/bus/auxiliary/auxiliary_common.c  | 419 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 142 ++++++++
 drivers/bus/auxiliary/meson.build         |  11 +
 drivers/bus/auxiliary/private.h           | 112 ++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
 drivers/bus/auxiliary/version.map         |   7 +
 drivers/bus/meson.build                   |   1 +
 10 files changed, 963 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..eaf691ca6a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..b335064963 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,13 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added auxiliary bus support.**
+
+  * Auxiliary bus provides a way to split function into child-devices
+    representing sub-domains of functionality. Each auxiliary device
+    represents a part of its parent functionality.
+  * Devargs syntax of auxiliary device: -a auxiliary:<name>[,args...]
+
 
 Removed Items
 -------------
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..91008bdc80
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,419 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AXILIARY_NAME, devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+/**
+ * Update a device being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary driver and device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it. */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	/* No initialization when marked as blocked, return without error. */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+		return -EINVAL;
+	}
+
+	dev->driver = dr;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+		      dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
+		      dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	const char **out = addr;
+
+	/* Allow dummy name to prevent bus scan. */
+	if (strlen(name) == 0)
+		return 0;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (dr->match(name))
+			break;
+	}
+	if (dr != NULL && addr != NULL)
+		*out = name;
+	return dr != NULL ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map)
+		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap)
+		return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..5a1b029839
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..22e0e20493
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?\n",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..3d5c7b0d4a
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+headers = files('rte_bus_auxiliary.h')
+sources = files('auxiliary_common.c',
+    'auxiliary_params.c')
+if is_linux
+    sources += files('linux/auxiliary.c')
+endif
+deps += ['kvargs']
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..3146fcd5d3
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+bool auxiliary_dev_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Update a device being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param aux_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_aux_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_aux_dev
+ *	AUXILIARY device to be added before exist_aux_dev
+ * @return void
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param aux_drv
+ *      auxiliary driver
+ * @param aux_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..94f0790ba0
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_BUS_AUXILIARY_H
+#define RTE_BUS_AUXILIARY_H
+
+/**
+ * @file
+ *
+ * Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+#define RTE_BUS_AXILIARY_NAME "auxiliary"
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(rte_auxiliary_match_t) (const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(rte_auxiliary_probe_t) (struct rte_auxiliary_driver *drv,
+				    struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t *match;         /**< Device match function. */
+	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
+	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */
+	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
+	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_ ##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&(auxiliary_drv));	\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BUS_AUXILIARY_H */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..a52260657c
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,7 @@
+EXPERIMENTAL {
+	global:
+
+	# added in 21.08
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 410058de3a..45eab5233d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1


^ permalink raw reply	[flat|nested] 28+ messages in thread

end of thread, other threads:[~2021-06-13 12:59 UTC | newest]

Thread overview: 28+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-03-11 13:01 [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus Xueming Li
2021-04-12  8:29 ` Xueming(Steven) Li
2021-04-13  3:23 ` [dpdk-dev] [PATCH v1] " Xueming Li
2021-04-13  8:49   ` Thomas Monjalon
2021-04-14  2:59   ` Wang, Haiyue
2021-04-14  8:17     ` Thomas Monjalon
2021-04-14  8:30       ` Wang, Haiyue
2021-04-14 15:49       ` Xueming(Steven) Li
2021-04-14 15:39     ` Xueming(Steven) Li
2021-04-14 16:13       ` Wang, Haiyue
2021-04-15  7:35   ` Wang, Haiyue
2021-04-15  7:46     ` Xueming(Steven) Li
2021-04-15  7:51       ` Wang, Haiyue
2021-04-15  7:55         ` Xueming(Steven) Li
2021-04-15  7:59           ` Thomas Monjalon
2021-04-15  8:06             ` Wang, Haiyue
2021-05-10 13:47   ` [dpdk-dev] [RFC v2] " Xueming Li
2021-05-11  9:47     ` Kinsella, Ray
2021-06-10  3:30       ` Xueming(Steven) Li
2021-06-08  7:53     ` Thomas Monjalon
2021-06-08  8:41       ` Wang, Haiyue
2021-06-10  6:29         ` Xueming(Steven) Li
2021-06-10 15:16           ` Wang, Haiyue
2021-06-10  6:30       ` Xueming(Steven) Li
2021-06-13  8:19     ` [dpdk-dev] [PATCH v3 1/2] devargs: add common key definition Xueming Li
2021-06-13  8:19     ` [dpdk-dev] [PATCH v3 2/2] bus/auxiliary: introduce auxiliary bus Xueming Li
2021-06-13 12:58     ` [dpdk-dev] [PATCH v4 1/2] devargs: add common key definition Xueming Li
2021-06-13 12:58     ` [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus Xueming Li

DPDK patches and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git