[dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus

[dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary_device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 MAINTAINERS                               |   5 +
 drivers/bus/auxiliary/auxiliary_common.c  | 410 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 144 ++++++++
 drivers/bus/auxiliary/meson.build         |  17 +
 drivers/bus/auxiliary/private.h           | 119 +++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 193 ++++++++++
 drivers/bus/auxiliary/version.map         |  11 +
 drivers/bus/meson.build                   |   2 +-
 9 files changed, 958 insertions(+), 1 deletion(-)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index e341bc81d5..00db01e471 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -514,6 +514,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..14b81dffce
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,410 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+#include <rte_vfio.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+
+#define SYSFS_AUXILIARY_DEVICES "/sys/bus/auxiliary/devices"
+
+int auxiliary_logtype_bus;
+
+const char *rte_auxiliary_get_sysfs_path(void)
+{
+	const char *path = NULL;
+
+#ifdef RTE_EXEC_ENV_LINUX
+	path = getenv("SYSFS_AUXILIARY_DEVICES");
+	if (path == NULL)
+		return SYSFS_AUXILIARY_DEVICES;
+#endif
+
+	return path;
+}
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+void
+auxiliary_on_scan(struct rte_auxiliary_device *dev)
+{
+	struct rte_devargs *devargs;
+
+	devargs = auxiliary_devargs_lookup(dev->name);
+	dev->device.devargs = devargs;
+}
+
+/*
+ * Match the auxiliary Driver and Device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
+		    const struct rte_auxiliary_device *auxiliary_dev)
+{
+	if (auxiliary_drv->match == NULL)
+		return false;
+	return auxiliary_drv->match(auxiliary_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	int ret;
+	bool dev_probed;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIAR_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+			dev->name, dev->device.numa_node);
+
+	/* no initialization when marked as blocked, return without error */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIAR_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIAR_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	dev_probed = rte_dev_is_probed(&dev->device);
+	if (dev_probed && !(dr->drv_flags & RTE_AUXILIARY_DRV_PROBE_AGAIN)) {
+		AUXILIAR_LOG(DEBUG, "  Device %s is already probed\n",
+				dev->device.name);
+		return -EEXIST;
+	}
+
+	AUXILIAR_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	if (!dev_probed) {
+		enum rte_iova_mode iova_mode;
+
+		iova_mode = rte_eal_iova_mode();
+		if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
+		    iova_mode != RTE_IOVA_VA) {
+			AUXILIAR_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+			return -EINVAL;
+		}
+
+		dev->driver = dr;
+	}
+
+	AUXILIAR_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+			dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (dev_probed)
+		return ret; /* no rollback if already succeeded earlier */
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_detach_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIAR_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		dev->name, dev->device.numa_node);
+
+	AUXILIAR_LOG(DEBUG, "  remove driver: %s %s\n",
+		dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	int rc = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIAR_LOG(ERR, "Requested device %s cannot be used\n",
+					dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	RTE_SET_USED(addr);
+	return auxiliary_exists(name) ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_auxiliary_dev,
+			    struct rte_auxiliary_device *new_auxiliary_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_auxiliary_dev, new_auxiliary_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_detach_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (!adev || !adev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (adev->driver->dma_map)
+		return adev->driver->dma_map(adev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (!adev || !adev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (adev->driver->dma_unmap)
+		return adev->driver->dma_unmap(adev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER(auxiliary_logtype_bus, bus.auxiliary, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..0e61020f3d
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	int ret;
+	const struct rte_kvargs *kvlist = _kvlist;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..7852e67b8c
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <rte_vfio.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+
+/**
+ * @file
+ * Linux auxiliary probing.
+ */
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	dev->device.bus = &auxiliary_bus.bus;
+	strlcpy(dev->name, name, sizeof(dev->name));
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 rte_auxiliary_get_sysfs_path(), name);
+	dir = opendir(rte_auxiliary_get_sysfs_path());
+	if (dir == NULL)
+		return true;
+	closedir(dir);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(rte_auxiliary_get_sysfs_path());
+	if (dir == NULL) {
+		AUXILIAR_LOG(ERR, "%s(): opendir failed: %s\n",
+			__func__, strerror(errno));
+		return -1;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+				rte_auxiliary_get_sysfs_path(), e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
+
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..0859620ed3
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Mellanox Technologies, Ltd
+
+if is_linux
+	headers = files('rte_bus_auxiliary.h')
+	sources = files('auxiliary_common.c',
+		'auxiliary_params.c',
+		'linux/auxiliary.c')
+	includes += include_directories('linux')
+	deps += ['kvargs']
+endif
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..b55827c74f
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_logtype_bus;
+
+#define AUXILIAR_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_logtype_bus, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+/**
+ * Test whether the auxiliary device exist
+ *
+ * @param name
+ *  Auxiliary device name
+ * @return
+ *  true on exists, false otherwise
+ */
+bool auxiliary_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Setup or update device when being scanned.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param auxiliary_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *auxiliary_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_auxiliary_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_auxiliary_dev
+ *	AUXILIARY device to be added before exist_auxiliary_dev
+ * @return void
+ */
+void auxiliary_insert_device(
+		struct rte_auxiliary_device *exist_auxiliary_dev,
+		struct rte_auxiliary_device *new_auxiliary_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param auxiliary_drv
+ *      auxiliary driver
+ * @param auxiliary_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
+		     const struct rte_auxiliary_device *auxiliary_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..b778a00749
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _RTE_BUS_AUXILIARY_H_
+#define _RTE_BUS_AUXILIARY_H_
+
+/**
+ * @file
+ *
+ * RTE Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ */
+typedef bool(auxiliary_match_t) (const char *);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ */
+typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
+				struct rte_auxiliary_device*);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ */
+typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
+				  uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+				    void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	auxiliary_match_t *match;            /**< Device match function. */
+	auxiliary_probe_t *probe;            /**< Device Probe function. */
+	auxiliary_remove_t *remove;          /**< Device Remove function. */
+	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
+	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+
+/** Device already probed can be probed again to check for new ports. */
+#define RTE_AUXILIARY_DRV_PROBE_AGAIN 0x0001
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Get sysfs pathname of auxiliary devices directory
+ *
+ * @return
+ *   Pathname of auxiliary devices directory
+ */
+__rte_experimental
+const char *rte_auxiliary_get_sysfs_path(void);
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&auxiliary_drv);		\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BUS_AUXILIARY_H_ */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..e012955b8f
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,11 @@
+DPDK_21 {
+	local: *;
+};
+
+EXPERIMENTAL {
+	global:
+
+	rte_auxiliary_get_sysfs_path;
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 2e7727af08..d581003f7d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
+drivers = ['auxiliary', 'dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
 std_deps = ['eal']
-- 
2.25.1

Re: [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

<snip>
> +/*
> + * Test whether the auxiliary device exist  */ bool
> +auxiliary_exists(const char *name) {
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +
> +	snprintf(dirname, sizeof(dirname), "%s/%s",
> +		 rte_auxiliary_get_sysfs_path(), name);
> +	dir = opendir(rte_auxiliary_get_sysfs_path());
> +	if (dir == NULL)
> +		return true;

Should return false here.

> +	closedir(dir);
> +	return false;

Return true, will fix in next version.

> +}
<snip>

[dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary_device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 MAINTAINERS                               |   5 +
 drivers/bus/auxiliary/auxiliary_common.c  | 391 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
 drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
 drivers/bus/auxiliary/meson.build         |  17 +
 drivers/bus/auxiliary/private.h           | 118 +++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
 drivers/bus/auxiliary/version.map         |  10 +
 drivers/bus/meson.build                   |   2 +-
 9 files changed, 927 insertions(+), 1 deletion(-)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 0ec5588540..fd0e0c60ad 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -514,6 +514,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..bb9ba12be9
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,391 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+
+int auxiliary_bus_logtype;
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+void
+auxiliary_on_scan(struct rte_auxiliary_device *dev)
+{
+	struct rte_devargs *devargs;
+
+	devargs = auxiliary_devargs_lookup(dev->name);
+	dev->device.devargs = devargs;
+}
+
+/*
+ * Match the auxiliary Driver and Device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
+		    const struct rte_auxiliary_device *auxiliary_dev)
+{
+	if (auxiliary_drv->match == NULL)
+		return false;
+	return auxiliary_drv->match(auxiliary_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	int ret;
+	enum rte_iova_mode iova_mode;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	/* no initialization when marked as blocked, return without error */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+		return -EINVAL;
+	}
+
+	dev->driver = dr;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+		      dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
+		      dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	int rc = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	const char **out = addr;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (dr->match(name))
+			break;
+	}
+	if (dr != NULL && addr != NULL)
+		*out = name;
+	return dr != NULL ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_auxiliary_dev,
+			    struct rte_auxiliary_device *new_auxiliary_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_auxiliary_dev, new_auxiliary_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (!adev || !adev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (adev->driver->dma_map)
+		return adev->driver->dma_map(adev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (!adev || !adev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (adev->driver->dma_unmap)
+		return adev->driver->dma_unmap(adev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..5fc84b1953
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	int ret;
+	const struct rte_kvargs *kvlist = _kvlist;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..7888b6c5da
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/**
+ * @file
+ * Linux auxiliary probing.
+ */
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL)
+		return true;
+	closedir(dir);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(ERR, "%s(): opendir failed: %s\n",
+			      __func__, strerror(errno));
+		return -1;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..d84305acc9
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+if is_linux
+	headers = files('rte_bus_auxiliary.h')
+	sources = files('auxiliary_common.c',
+		'auxiliary_params.c',
+		'linux/auxiliary.c')
+	includes += include_directories('linux')
+	deps += ['kvargs']
+endif
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..b1b97bc26e
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+/**
+ * Test whether the auxiliary device exist
+ *
+ * @param name
+ *  Auxiliary device name
+ * @return
+ *  true on exists, false otherwise
+ */
+bool auxiliary_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Setup or update device when being scanned.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param auxiliary_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *auxiliary_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_auxiliary_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_auxiliary_dev
+ *	AUXILIARY device to be added before exist_auxiliary_dev
+ * @return void
+ */
+void auxiliary_insert_device(
+		struct rte_auxiliary_device *exist_auxiliary_dev,
+		struct rte_auxiliary_device *new_auxiliary_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param auxiliary_drv
+ *      auxiliary driver
+ * @param auxiliary_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
+		     const struct rte_auxiliary_device *auxiliary_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..816d65047b
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _RTE_BUS_AUXILIARY_H_
+#define _RTE_BUS_AUXILIARY_H_
+
+/**
+ * @file
+ *
+ * RTE Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ */
+typedef bool(auxiliary_match_t) (const char *);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ */
+typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
+				struct rte_auxiliary_device*);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ */
+typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
+				  uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+				    void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	auxiliary_match_t *match;            /**< Device match function. */
+	auxiliary_probe_t *probe;            /**< Device Probe function. */
+	auxiliary_remove_t *remove;          /**< Device Remove function. */
+	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
+	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&auxiliary_drv);		\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BUS_AUXILIARY_H_ */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..3d270baea7
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,10 @@
+DPDK_21 {
+	local: *;
+};
+
+EXPERIMENTAL {
+	global:
+
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 2e7727af08..d581003f7d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2017 Intel Corporation
 
-drivers = ['dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
+drivers = ['auxiliary', 'dpaa', 'fslmc', 'ifpga', 'pci', 'vdev', 'vmbus']
 std_deps = ['eal']
-- 
2.25.1

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

I would like to Cc those who are interested in the auxiliary bus,
but it's difficult to know who will implement it.
Can we expect some reviews?


13/04/2021 05:23, Xueming Li:
> Auxiliary [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary_device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Wang, Haiyue]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> Sent: Tuesday, April 13, 2021 11:23
> To: Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>;
> Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> Auxiliary [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary_device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> ---
>  MAINTAINERS                               |   5 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 391 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
>  drivers/bus/auxiliary/meson.build         |  17 +
>  drivers/bus/auxiliary/private.h           | 118 +++++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
>  drivers/bus/auxiliary/version.map         |  10 +
>  drivers/bus/meson.build                   |   2 +-
>  9 files changed, 927 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 


> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
> @@ -0,0 +1,391 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdbool.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <rte_errno.h>
> +#include <rte_interrupts.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_eal.h>
> +#include <rte_eal_paging.h>
> +#include <rte_string_fns.h>
> +#include <rte_common.h>
> +#include <rte_devargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +
> +int auxiliary_bus_logtype;
> +
> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name)
> +{
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
> +		if (strcmp(devargs->name, name) == 0)
> +			return devargs;
> +	}
> +	return NULL;
> +}
> +
> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_devargs *devargs;
> +
> +	devargs = auxiliary_devargs_lookup(dev->name);
> +	dev->device.devargs = devargs;

Can be simple as:

dev->device.devargs = auxiliary_devargs_lookup(dev->name);

> +}
> +
> +/*
> + * Match the auxiliary Driver and Device using driver function.
> + */
> +bool
> +auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
> +		    const struct rte_auxiliary_device *auxiliary_dev)

How about these auxiliary variable name style ?

const struct rte_auxiliary_driver *aux_drv,
const struct rte_auxiliary_device *aux_dev

> +{
> +	if (auxiliary_drv->match == NULL)
> +		return false;
> +	return auxiliary_drv->match(auxiliary_dev->name);
> +}
> +
> +/*
> + * Call the probe() function of the driver.
> + */
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> +			       struct rte_auxiliary_device *dev)
> +{
> +	int ret;
> +	enum rte_iova_mode iova_mode;
> +

RCT style ?
	enum rte_iova_mode iova_mode;
	int ret;


> +	if ((dr == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	/* The device is not blocked; Check if driver supports it */
> +	if (!auxiliary_match(dr, dev))
> +		/* Match of device and driver failed */
> +		return 1;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	/* no initialization when marked as blocked, return without error */
> +	if (dev->device.devargs != NULL &&
> +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> +		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
> +		return -1;
> +	}
> +
> +	if (dev->device.numa_node < 0) {
> +		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
> +		dev->device.numa_node = 0;
> +	}
> +
> +	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
> +
> +	iova_mode = rte_eal_iova_mode();
> +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&

'(dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)' should work, no need '> 0'

> +	    iova_mode != RTE_IOVA_VA) {
> +		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
> +		return -EINVAL;
> +	}
> +
> +	dev->driver = dr;
> +
> +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
> +		      dr->driver.name, dev->name, dev->device.numa_node);
> +	ret = dr->probe(dr, dev);
> +	if (ret)
> +		dev->driver = NULL;
> +	else
> +		dev->device.driver = &dr->driver;
> +
> +	return ret;
> +}
> +
> +/*
> + * Call the remove() function of the driver.
> + */
> +static int
> +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr;
> +	int ret = 0;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	dr = dev->driver;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
> +		      dev->name, dr->driver.name);
> +
> +	if (dr->remove) {
> +		ret = dr->remove(dev);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	/* clear driver structure */
> +	dev->driver = NULL;
> +	dev->device.driver = NULL;
> +
> +	return 0;
> +}
> +
> +/*
> + * Call the probe() function of all registered driver for the given device.
> + * Return < 0 if initialization failed.
> + * Return 1 if no driver is found for this device.
> + */
> +static int
> +auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr = NULL;
> +	int rc = 0;

These two variables need no initialization.

> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
> +		if (!dr->match(dev->name))
> +			continue;
> +
> +		rc = rte_auxiliary_probe_one_driver(dr, dev);
> +		if (rc < 0)
> +			/* negative value is an error */
> +			return rc;
> +		if (rc > 0)
> +			/* positive value means driver doesn't support it */
> +			continue;
> +		return 0;
> +	}
> +	return 1;
> +}
> +


> +static int
> +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> +{
> +	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);

How about to use 'aux_dev', instead of 'adev' ?

> +
> +	if (!adev || !adev->driver) {

' RTE_DEV_TO_AUXILIARY' is container of 'dev', so it should check 'dev != NULL',
not '!adev'. ; -)


> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (adev->driver->dma_map)
> +		return adev->driver->dma_map(adev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;
> +}
> +
> +static int
> +auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
> +		    size_t len)
> +{
> +	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (!adev || !adev->driver) {

The same comment as 'auxiliary_dma_map'

> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (adev->driver->dma_unmap)
> +		return adev->driver->dma_unmap(adev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;
> +}
> +


> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_params.c
> @@ -0,0 +1,58 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +
> +#include <rte_bus.h>
> +#include <rte_dev.h>
> +#include <rte_errno.h>
> +#include <rte_kvargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +enum auxiliary_params {
> +	RTE_AUXILIARY_PARAM_NAME,
> +};
> +
> +static const char * const auxiliary_params_keys[] = {
> +	[RTE_AUXILIARY_PARAM_NAME] = "name",
> +};
> +
> +static int
> +auxiliary_dev_match(const struct rte_device *dev,
> +	      const void *_kvlist)
> +{
> +	int ret;
> +	const struct rte_kvargs *kvlist = _kvlist;
> +

RCT.

> +	ret = rte_kvargs_process(kvlist,
> +			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
> +			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
> +
> +	return ret != 0 ? -1 : 0;
> +}
> +
> +void *
> +auxiliary_dev_iterate(const void *start,
> +		    const char *str,
> +		    const struct rte_dev_iterator *it __rte_unused)
> +{
> +	rte_bus_find_device_t find_device;
> +	struct rte_kvargs *kvargs = NULL;
> +	struct rte_device *dev;
> +
> +	if (str != NULL) {
> +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> +		if (kvargs == NULL) {
> +			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
> +			rte_errno = EINVAL;
> +			return NULL;
> +		}
> +	}
> +	find_device = auxiliary_bus.bus.find_device;
> +	dev = find_device(start, auxiliary_dev_match, kvargs);
> +	rte_kvargs_free(kvargs);
> +	return dev;
> +}


> diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
> new file mode 100644
> index 0000000000..7888b6c5da
> --- /dev/null
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
                              ^
                              |
Seems no need to add one more directory 'linux' layer, as the meson said "linux only".


> @@ -0,0 +1,147 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +#include <dirent.h>
> +
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <eal_filesystem.h>
> +
> +#include "../rte_bus_auxiliary.h"
> +#include "../private.h"
> +
> +#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
> +
> +/**
> + * @file
> + * Linux auxiliary probing.
> + */
> +
> +/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
> +static int
> +auxiliary_scan_one(const char *dirname, const char *name)
> +{
> +	struct rte_auxiliary_device *dev;
> +	struct rte_auxiliary_device *dev2;
> +	char filename[PATH_MAX];
> +	unsigned long tmp;
> +	int ret;
> +
> +	dev = malloc(sizeof(*dev));
> +	if (dev == NULL)
> +		return -1;
> +
> +	memset(dev, 0, sizeof(*dev));
> +	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
> +		free(dev);
> +		return -1;
> +	}
> +	dev->device.name = dev->name;
> +	dev->device.bus = &auxiliary_bus.bus;
> +
> +	/* Get numa node, default to 0 if not present */
> +	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
> +		 dirname, name);
> +	if (access(filename, F_OK) != -1) {
> +		if (eal_parse_sysfs_value(filename, &tmp) == 0)
> +			dev->device.numa_node = tmp;
> +		else
> +			dev->device.numa_node = -1;
> +	} else {
> +		dev->device.numa_node = 0;
> +	}
> +
> +	auxiliary_on_scan(dev);
> +
> +	/* Device is valid, add in list (sorted) */
> +	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
> +		ret = strcmp(dev->name, dev2->name);
> +		if (ret > 0)
> +			continue;
> +		if (ret < 0) {
> +			auxiliary_insert_device(dev2, dev);
> +		} else { /* already registered */
> +			if (rte_dev_is_probed(&dev2->device) &&
> +			    dev2->device.devargs != dev->device.devargs) {
> +				/* To probe device with new devargs. */
> +				rte_devargs_remove(dev2->device.devargs);
> +				auxiliary_on_scan(dev2);
> +			}
> +			free(dev);
> +		}
> +		return 0;
> +	}
> +	auxiliary_add_device(dev);
> +	return 0;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist
> + */
> +bool
> +auxiliary_exists(const char *name)

is_auxiliary_support() ?

> +{
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +
> +	snprintf(dirname, sizeof(dirname), "%s/%s",
> +		 AUXILIARY_SYSFS_PATH, name);
> +	dir = opendir(AUXILIARY_SYSFS_PATH);
> +	if (dir == NULL)
> +		return true;

false

> +	closedir(dir);
> +	return false;

true



> --
> 2.25.1

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

14/04/2021 04:59, Wang, Haiyue:
> From: Xueming Li
[...]
> > +void
> > +auxiliary_on_scan(struct rte_auxiliary_device *dev)
> > +{
> > +	struct rte_devargs *devargs;
> > +
> > +	devargs = auxiliary_devargs_lookup(dev->name);
> > +	dev->device.devargs = devargs;
> 
> Can be simple as:
> 
> dev->device.devargs = auxiliary_devargs_lookup(dev->name);
> 
> > +}
> > +
> > +/*
> > + * Match the auxiliary Driver and Device using driver function.
> > + */
> > +bool
> > +auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
> > +		    const struct rte_auxiliary_device *auxiliary_dev)
> 
> How about these auxiliary variable name style ?
> 
> const struct rte_auxiliary_driver *aux_drv,
> const struct rte_auxiliary_device *aux_dev

+1

[...]
> > +static int
> > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > +			       struct rte_auxiliary_device *dev)
> > +{
> > +	int ret;
> > +	enum rte_iova_mode iova_mode;
> > +
> 
> RCT style ?
> 	enum rte_iova_mode iova_mode;
> 	int ret;

I don't see the benefit of reverse christmas tree.

> > +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> 
> '(dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)' should work, no need '> 0'

Yes it's acceptable to consider bit testing as a boolean.

[...]
> > +static int
> > +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> > +{
> > +	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
> 
> How about to use 'aux_dev', instead of 'adev' ?
> 
> > +
> > +	if (!adev || !adev->driver) {
> 
> ' RTE_DEV_TO_AUXILIARY' is container of 'dev', so it should check 'dev != NULL',
> not '!adev'. ; -)

Yes and should be explicit NULL comparison.

[...]
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
>                               ^
>                               |
> Seems no need to add one more directory 'linux' layer, as the meson said "linux only".

I disagree.
Linux sub-directory is more explicit.
And who knows? There could be an implementation on other OSes in future.

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Wang, Haiyue]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, April 14, 2021 16:18
> To: Xueming Li <xuemingl@nvidia.com>; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>
> Subject: Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> 14/04/2021 04:59, Wang, Haiyue:
> > From: Xueming Li


> 
> [...]
> > > +static int
> > > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > > +			       struct rte_auxiliary_device *dev)
> > > +{
> > > +	int ret;
> > > +	enum rte_iova_mode iova_mode;
> > > +
> >
> > RCT style ?
> > 	enum rte_iova_mode iova_mode;
> > 	int ret;
> 
> I don't see the benefit of reverse christmas tree.
> 

I'm impacted by kernel code style ;-)


> > > +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> [...]
> > > --- /dev/null
> > > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> >                               ^
> >                               |
> > Seems no need to add one more directory 'linux' layer, as the meson said "linux only".
> 
> I disagree.
> Linux sub-directory is more explicit.
> And who knows? There could be an implementation on other OSes in future.

Make sense.

> 
> 

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

Hi Haiyue,

> > +
> > +/*
> > + * Test whether the auxiliary device exist  */ bool
> > +auxiliary_exists(const char *name)
> 
> is_auxiliary_support() ?

This function test whether an auxiliary device exists, name parameter is device name.
I think auxiliary_dev_exists(name) make more sense.

Agree with other comments, thanks!

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, April 14, 2021 4:18 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; Wang, Haiyue <haiyue.wang@intel.com>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>
> Subject: Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> 14/04/2021 04:59, Wang, Haiyue:
> > From: Xueming Li
> [...]
> > > +void
> > > +auxiliary_on_scan(struct rte_auxiliary_device *dev) {
> > > +	struct rte_devargs *devargs;
> > > +
> > > +	devargs = auxiliary_devargs_lookup(dev->name);
> > > +	dev->device.devargs = devargs;
> >
> > Can be simple as:
> >
> > dev->device.devargs = auxiliary_devargs_lookup(dev->name);
> >
> > > +}
> > > +
> > > +/*
> > > + * Match the auxiliary Driver and Device using driver function.
> > > + */
> > > +bool
> > > +auxiliary_match(const struct rte_auxiliary_driver *auxiliary_drv,
> > > +		    const struct rte_auxiliary_device *auxiliary_dev)
> >
> > How about these auxiliary variable name style ?
> >
> > const struct rte_auxiliary_driver *aux_drv, const struct
> > rte_auxiliary_device *aux_dev
> 
> +1
> 
> [...]
> > > +static int
> > > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > > +			       struct rte_auxiliary_device *dev) {
> > > +	int ret;
> > > +	enum rte_iova_mode iova_mode;
> > > +
> >
> > RCT style ?
> > 	enum rte_iova_mode iova_mode;
> > 	int ret;
> 
> I don't see the benefit of reverse christmas tree.
> 
> > > +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> >
> > '(dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)' should work, no need '> 0'
> 
> Yes it's acceptable to consider bit testing as a boolean.
> 
> [...]
> > > +static int
> > > +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t
> > > +iova, size_t len) {
> > > +	struct rte_auxiliary_device *adev = RTE_DEV_TO_AUXILIARY(dev);
> >
> > How about to use 'aux_dev', instead of 'adev' ?
> >
> > > +
> > > +	if (!adev || !adev->driver) {
> >
> > ' RTE_DEV_TO_AUXILIARY' is container of 'dev', so it should check 'dev
> > != NULL', not '!adev'. ; -)
> 
> Yes and should be explicit NULL comparison.
> 
> [...]
> > > --- /dev/null
> > > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> >                               ^
> >                               |
> > Seems no need to add one more directory 'linux' layer, as the meson said "linux only".
> 
> I disagree.
> Linux sub-directory is more explicit.
> And who knows? There could be an implementation on other OSes in future.

This reminds me to change meson, allows bus always available to avoid compilation error,
also need to add stubs with __rte_weak for all functions in common file.

> 
> 

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Wang, Haiyue]

> -----Original Message-----
> From: Xueming(Steven) Li <xuemingl@nvidia.com>
> Sent: Wednesday, April 14, 2021 23:39
> To: Wang, Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> Hi Haiyue,
> 
> > > +
> > > +/*
> > > + * Test whether the auxiliary device exist  */ bool
> > > +auxiliary_exists(const char *name)
> >
> > is_auxiliary_support() ?
> 
> This function test whether an auxiliary device exists, name parameter is device name.
> I think auxiliary_dev_exists(name) make more sense.

More clear now. ;-)

> 
> Agree with other comments, thanks!

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Wang, Haiyue]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> Sent: Tuesday, April 13, 2021 11:23
> To: Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>;
> Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> Auxiliary [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary_device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> ---
>  MAINTAINERS                               |   5 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 391 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
>  drivers/bus/auxiliary/meson.build         |  17 +
>  drivers/bus/auxiliary/private.h           | 118 +++++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
>  drivers/bus/auxiliary/version.map         |  10 +
>  drivers/bus/meson.build                   |   2 +-
>  9 files changed, 927 insertions(+), 1 deletion(-)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 


> --- /dev/null
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> @@ -0,0 +1,180 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef _RTE_BUS_AUXILIARY_H_
> +#define _RTE_BUS_AUXILIARY_H_
> +
> +/**
> + * @file
> + *
> + * RTE Auxiliary Bus Interface.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <sys/queue.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +
> +#include <rte_debug.h>
> +#include <rte_interrupts.h>
> +#include <rte_dev.h>
> +#include <rte_bus.h>
> +#include <rte_kvargs.h>
> +
> +/* Forward declarations */
> +struct rte_auxiliary_driver;
> +struct rte_auxiliary_bus;
> +struct rte_auxiliary_device;
> +
> +/**
> + * Match function for the driver to decide if device can be handled.
> + */
> +typedef bool(auxiliary_match_t) (const char *);
> +
> +/**
> + * Initialization function for the driver called during auxiliary probing.
> + */
> +typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
> +				struct rte_auxiliary_device*);
> +
> +/**
> + * Uninitialization function for the driver called during hotplugging.
> + */
> +typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
> +
> +/**
> + * Driver-specific DMA mapping. After a successful call the device
> + * will be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be mapped.
> + * @param iova
> + *   Starting IOVA address of memory to be mapped.
> + * @param len
> + *   Length of memory segment being mapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
> +				  uint64_t iova, size_t len);
> +
> +/**
> + * Driver-specific DMA un-mapping. After a successful call the device
> + * will not be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be unmapped.
> + * @param iova
> + *   Starting IOVA address of memory to be unmapped.
> + * @param len
> + *   Length of memory segment being unmapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> +				    void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * A structure describing an auxiliary device.
> + */
> +struct rte_auxiliary_device {
> +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> +	struct rte_device device;                 /**< Inherit core device */
> +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> +};
> +
> +/** List of auxiliary devices */
> +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> +/** List of auxiliary drivers */
> +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> +
> +/**
> + * Structure describing the auxiliary bus
> + */
> +struct rte_auxiliary_bus {
> +	struct rte_bus bus;                  /**< Inherit the generic class */
> +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
> +};
> +
> +/**
> + * A structure describing an auxiliary driver.
> + */
> +struct rte_auxiliary_driver {
> +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> +	struct rte_driver driver;            /**< Inherit core driver. */
> +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> +	auxiliary_match_t *match;            /**< Device match function. */
> +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */

These API type can be pointer type defined, then no need "*":

typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
				    void *addr, uint64_t iova, size_t len);

auxiliary_dma_unmap_t dma_unmap;

Like:
https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759-1-thomas@monjalon.net/

typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
				 void *addr, uint64_t iova, size_t len);

> --
> 2.25.1

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Wang, Haiyue <haiyue.wang@intel.com>
> Sent: Thursday, April 15, 2021 3:36 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman
> <nhorman@tuxdriver.com>
> Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > Sent: Tuesday, April 13, 2021 11:23
> > To: Thomas Monjalon <thomas@monjalon.net>
> > Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso <asafp@nvidia.com>;
> > Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil
> > Horman <nhorman@tuxdriver.com>
> > Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> >
> > Auxiliary [1] provides a way to split function into child-devices
> > representing sub-domains of functionality. Each auxiliary_device
> > represents a part of its parent functionality.
> >
> > Auxiliary device is identified by unique device name, sysfs path:
> >   /sys/bus/auxiliary/devices/<name>
> >
> > [1] kernel auxiliary bus document:
> > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > ---
> >  MAINTAINERS                               |   5 +
> >  drivers/bus/auxiliary/auxiliary_common.c  | 391
> > ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
> >  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
> >  drivers/bus/auxiliary/meson.build         |  17 +
> >  drivers/bus/auxiliary/private.h           | 118 +++++++
> >  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
> >  drivers/bus/auxiliary/version.map         |  10 +
> >  drivers/bus/meson.build                   |   2 +-
> >  9 files changed, 927 insertions(+), 1 deletion(-)  create mode 100644
> > drivers/bus/auxiliary/auxiliary_common.c
> >  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> >  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> >  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> > 100644 drivers/bus/auxiliary/private.h  create mode 100644
> > drivers/bus/auxiliary/rte_bus_auxiliary.h
> >  create mode 100644 drivers/bus/auxiliary/version.map
> >
> 
> 
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > @@ -0,0 +1,180 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2021 Mellanox Technologies, Ltd  */
> > +
> > +#ifndef _RTE_BUS_AUXILIARY_H_
> > +#define _RTE_BUS_AUXILIARY_H_
> > +
> > +/**
> > + * @file
> > + *
> > + * RTE Auxiliary Bus Interface.
> > + */
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <limits.h>
> > +#include <errno.h>
> > +#include <sys/queue.h>
> > +#include <stdint.h>
> > +#include <inttypes.h>
> > +
> > +#include <rte_debug.h>
> > +#include <rte_interrupts.h>
> > +#include <rte_dev.h>
> > +#include <rte_bus.h>
> > +#include <rte_kvargs.h>
> > +
> > +/* Forward declarations */
> > +struct rte_auxiliary_driver;
> > +struct rte_auxiliary_bus;
> > +struct rte_auxiliary_device;
> > +
> > +/**
> > + * Match function for the driver to decide if device can be handled.
> > + */
> > +typedef bool(auxiliary_match_t) (const char *);
> > +
> > +/**
> > + * Initialization function for the driver called during auxiliary probing.
> > + */
> > +typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
> > +				struct rte_auxiliary_device*);
> > +
> > +/**
> > + * Uninitialization function for the driver called during hotplugging.
> > + */
> > +typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
> > +
> > +/**
> > + * Driver-specific DMA mapping. After a successful call the device
> > + * will be able to read/write from/to this segment.
> > + *
> > + * @param dev
> > + *   Pointer to the auxiliary device.
> > + * @param addr
> > + *   Starting virtual address of memory to be mapped.
> > + * @param iova
> > + *   Starting IOVA address of memory to be mapped.
> > + * @param len
> > + *   Length of memory segment being mapped.
> > + * @return
> > + *   - 0 On success.
> > + *   - Negative value and rte_errno is set otherwise.
> > + */
> > +typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
> > +				  uint64_t iova, size_t len);
> > +
> > +/**
> > + * Driver-specific DMA un-mapping. After a successful call the device
> > + * will not be able to read/write from/to this segment.
> > + *
> > + * @param dev
> > + *   Pointer to the auxiliary device.
> > + * @param addr
> > + *   Starting virtual address of memory to be unmapped.
> > + * @param iova
> > + *   Starting IOVA address of memory to be unmapped.
> > + * @param len
> > + *   Length of memory segment being unmapped.
> > + * @return
> > + *   - 0 On success.
> > + *   - Negative value and rte_errno is set otherwise.
> > + */
> > +typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > +				    void *addr, uint64_t iova, size_t len);
> > +
> > +/**
> > + * A structure describing an auxiliary device.
> > + */
> > +struct rte_auxiliary_device {
> > +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> > +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> > +	struct rte_device device;                 /**< Inherit core device */
> > +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> > +};
> > +
> > +/** List of auxiliary devices */
> > +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> > +/** List of auxiliary drivers */
> > +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> > +
> > +/**
> > + * Structure describing the auxiliary bus  */ struct
> > +rte_auxiliary_bus {
> > +	struct rte_bus bus;                  /**< Inherit the generic class */
> > +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> > +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers
> > +*/ };
> > +
> > +/**
> > + * A structure describing an auxiliary driver.
> > + */
> > +struct rte_auxiliary_driver {
> > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > +	struct rte_driver driver;            /**< Inherit core driver. */
> > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > +	auxiliary_match_t *match;            /**< Device match function. */
> > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> 
> These API type can be pointer type defined, then no need "*":
> 
> typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> 				    void *addr, uint64_t iova, size_t len);
> 
> auxiliary_dma_unmap_t dma_unmap;
> 
> Like:
> https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759-1-thomas@monjalon.net/
> 
> typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> 				 void *addr, uint64_t iova, size_t len);

Thanks, is there a reason to prefer pointer type? 

Thoma's patch looks good, will rebase on it once accepted.

> 
> > --
> > 2.25.1

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Wang, Haiyue]

> -----Original Message-----
> From: Xueming(Steven) Li <xuemingl@nvidia.com>
> Sent: Thursday, April 15, 2021 15:46
> To: Wang, Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> 
> > -----Original Message-----
> > From: Wang, Haiyue <haiyue.wang@intel.com>
> > Sent: Thursday, April 15, 2021 3:36 PM
> > To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> > Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; Neil Horman
> > <nhorman@tuxdriver.com>
> > Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> >
> > > -----Original Message-----
> > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > Sent: Tuesday, April 13, 2021 11:23
> > > To: Thomas Monjalon <thomas@monjalon.net>
> > > Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso <asafp@nvidia.com>;
> > > Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil
> > > Horman <nhorman@tuxdriver.com>
> > > Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> > >
> > > Auxiliary [1] provides a way to split function into child-devices
> > > representing sub-domains of functionality. Each auxiliary_device
> > > represents a part of its parent functionality.
> > >
> > > Auxiliary device is identified by unique device name, sysfs path:
> > >   /sys/bus/auxiliary/devices/<name>
> > >
> > > [1] kernel auxiliary bus document:
> > > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> > >
> > > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > > ---
> > >  MAINTAINERS                               |   5 +
> > >  drivers/bus/auxiliary/auxiliary_common.c  | 391
> > > ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c  |  58 ++++
> > >  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
> > >  drivers/bus/auxiliary/meson.build         |  17 +
> > >  drivers/bus/auxiliary/private.h           | 118 +++++++
> > >  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
> > >  drivers/bus/auxiliary/version.map         |  10 +
> > >  drivers/bus/meson.build                   |   2 +-
> > >  9 files changed, 927 insertions(+), 1 deletion(-)  create mode 100644
> > > drivers/bus/auxiliary/auxiliary_common.c
> > >  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> > >  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> > >  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> > > 100644 drivers/bus/auxiliary/private.h  create mode 100644
> > > drivers/bus/auxiliary/rte_bus_auxiliary.h
> > >  create mode 100644 drivers/bus/auxiliary/version.map
> > >
> >
> >
> > > --- /dev/null
> > > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > > @@ -0,0 +1,180 @@
> > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > + * Copyright 2021 Mellanox Technologies, Ltd  */
> > > +
> > > +#ifndef _RTE_BUS_AUXILIARY_H_
> > > +#define _RTE_BUS_AUXILIARY_H_
> > > +
> > > +/**
> > > + * @file
> > > + *
> > > + * RTE Auxiliary Bus Interface.
> > > + */
> > > +
> > > +#ifdef __cplusplus
> > > +extern "C" {
> > > +#endif
> > > +
> > > +#include <stdio.h>
> > > +#include <stdlib.h>
> > > +#include <limits.h>
> > > +#include <errno.h>
> > > +#include <sys/queue.h>
> > > +#include <stdint.h>
> > > +#include <inttypes.h>
> > > +
> > > +#include <rte_debug.h>
> > > +#include <rte_interrupts.h>
> > > +#include <rte_dev.h>
> > > +#include <rte_bus.h>
> > > +#include <rte_kvargs.h>
> > > +
> > > +/* Forward declarations */
> > > +struct rte_auxiliary_driver;
> > > +struct rte_auxiliary_bus;
> > > +struct rte_auxiliary_device;
> > > +
> > > +/**
> > > + * Match function for the driver to decide if device can be handled.
> > > + */
> > > +typedef bool(auxiliary_match_t) (const char *);
> > > +
> > > +/**
> > > + * Initialization function for the driver called during auxiliary probing.
> > > + */
> > > +typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
> > > +				struct rte_auxiliary_device*);
> > > +
> > > +/**
> > > + * Uninitialization function for the driver called during hotplugging.
> > > + */
> > > +typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
> > > +
> > > +/**
> > > + * Driver-specific DMA mapping. After a successful call the device
> > > + * will be able to read/write from/to this segment.
> > > + *
> > > + * @param dev
> > > + *   Pointer to the auxiliary device.
> > > + * @param addr
> > > + *   Starting virtual address of memory to be mapped.
> > > + * @param iova
> > > + *   Starting IOVA address of memory to be mapped.
> > > + * @param len
> > > + *   Length of memory segment being mapped.
> > > + * @return
> > > + *   - 0 On success.
> > > + *   - Negative value and rte_errno is set otherwise.
> > > + */
> > > +typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
> > > +				  uint64_t iova, size_t len);
> > > +
> > > +/**
> > > + * Driver-specific DMA un-mapping. After a successful call the device
> > > + * will not be able to read/write from/to this segment.
> > > + *
> > > + * @param dev
> > > + *   Pointer to the auxiliary device.
> > > + * @param addr
> > > + *   Starting virtual address of memory to be unmapped.
> > > + * @param iova
> > > + *   Starting IOVA address of memory to be unmapped.
> > > + * @param len
> > > + *   Length of memory segment being unmapped.
> > > + * @return
> > > + *   - 0 On success.
> > > + *   - Negative value and rte_errno is set otherwise.
> > > + */
> > > +typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > +				    void *addr, uint64_t iova, size_t len);
> > > +
> > > +/**
> > > + * A structure describing an auxiliary device.
> > > + */
> > > +struct rte_auxiliary_device {
> > > +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> > > +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> > > +	struct rte_device device;                 /**< Inherit core device */
> > > +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> > > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> > > +};
> > > +
> > > +/** List of auxiliary devices */
> > > +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> > > +/** List of auxiliary drivers */
> > > +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> > > +
> > > +/**
> > > + * Structure describing the auxiliary bus  */ struct
> > > +rte_auxiliary_bus {
> > > +	struct rte_bus bus;                  /**< Inherit the generic class */
> > > +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> > > +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers
> > > +*/ };
> > > +
> > > +/**
> > > + * A structure describing an auxiliary driver.
> > > + */
> > > +struct rte_auxiliary_driver {
> > > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > > +	struct rte_driver driver;            /**< Inherit core driver. */
> > > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > > +	auxiliary_match_t *match;            /**< Device match function. */
> > > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> >
> > These API type can be pointer type defined, then no need "*":
> >
> > typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > 				    void *addr, uint64_t iova, size_t len);
> >
> > auxiliary_dma_unmap_t dma_unmap;
> >
> > Like:
> > https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759-1-thomas@monjalon.net/
> >
> > typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> > 				 void *addr, uint64_t iova, size_t len);
> 
> Thanks, is there a reason to prefer pointer type?

Good practice to make code beautiful ? ;-)

> 
> Thoma's patch looks good, will rebase on it once accepted.

I mean the function type is defined as pointer type.

> 
> >
> > > --
> > > 2.25.1

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Wang, Haiyue <haiyue.wang@intel.com>
> Sent: Thursday, April 15, 2021 3:52 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman
> <nhorman@tuxdriver.com>
> Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> > -----Original Message-----
> > From: Xueming(Steven) Li <xuemingl@nvidia.com>
> > Sent: Thursday, April 15, 2021 15:46
> > To: Wang, Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon
> > <thomas@monjalon.net>
> > Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit
> > <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman
> > <nhorman@tuxdriver.com>
> > Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary
> > bus
> >
> >
> > > -----Original Message-----
> > > From: Wang, Haiyue <haiyue.wang@intel.com>
> > > Sent: Thursday, April 15, 2021 3:36 PM
> > > To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas
> > > Monjalon <thomas@monjalon.net>
> > > Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit
> > > <parav@nvidia.com>; Ray Kinsella
> > <mdr@ashroe.eu>; Neil Horman
> > > <nhorman@tuxdriver.com>
> > > Subject: RE: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce
> > > auxiliary bus
> > >
> > > > -----Original Message-----
> > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > > Sent: Tuesday, April 13, 2021 11:23
> > > > To: Thomas Monjalon <thomas@monjalon.net>
> > > > Cc: dev@dpdk.org; xuemingl@nvidia.com; Asaf Penso
> > > > <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> > > > <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> > > > Subject: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary
> > > > bus
> > > >
> > > > Auxiliary [1] provides a way to split function into child-devices
> > > > representing sub-domains of functionality. Each auxiliary_device
> > > > represents a part of its parent functionality.
> > > >
> > > > Auxiliary device is identified by unique device name, sysfs path:
> > > >   /sys/bus/auxiliary/devices/<name>
> > > >
> > > > [1] kernel auxiliary bus document:
> > > > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.ht
> > > > ml
> > > >
> > > > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > > > ---
> > > >  MAINTAINERS                               |   5 +
> > > >  drivers/bus/auxiliary/auxiliary_common.c  | 391
> > > > ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c
> > > > ++++++++++++++++++++++ |  58 ++++
> > > >  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
> > > >  drivers/bus/auxiliary/meson.build         |  17 +
> > > >  drivers/bus/auxiliary/private.h           | 118 +++++++
> > > >  drivers/bus/auxiliary/rte_bus_auxiliary.h | 180 ++++++++++
> > > >  drivers/bus/auxiliary/version.map         |  10 +
> > > >  drivers/bus/meson.build                   |   2 +-
> > > >  9 files changed, 927 insertions(+), 1 deletion(-)  create mode
> > > > 100644 drivers/bus/auxiliary/auxiliary_common.c
> > > >  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> > > >  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> > > >  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> > > > 100644 drivers/bus/auxiliary/private.h  create mode 100644
> > > > drivers/bus/auxiliary/rte_bus_auxiliary.h
> > > >  create mode 100644 drivers/bus/auxiliary/version.map
> > > >
> > >
> > >
> > > > --- /dev/null
> > > > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > > > @@ -0,0 +1,180 @@
> > > > +/* SPDX-License-Identifier: BSD-3-Clause
> > > > + * Copyright 2021 Mellanox Technologies, Ltd  */
> > > > +
> > > > +#ifndef _RTE_BUS_AUXILIARY_H_
> > > > +#define _RTE_BUS_AUXILIARY_H_
> > > > +
> > > > +/**
> > > > + * @file
> > > > + *
> > > > + * RTE Auxiliary Bus Interface.
> > > > + */
> > > > +
> > > > +#ifdef __cplusplus
> > > > +extern "C" {
> > > > +#endif
> > > > +
> > > > +#include <stdio.h>
> > > > +#include <stdlib.h>
> > > > +#include <limits.h>
> > > > +#include <errno.h>
> > > > +#include <sys/queue.h>
> > > > +#include <stdint.h>
> > > > +#include <inttypes.h>
> > > > +
> > > > +#include <rte_debug.h>
> > > > +#include <rte_interrupts.h>
> > > > +#include <rte_dev.h>
> > > > +#include <rte_bus.h>
> > > > +#include <rte_kvargs.h>
> > > > +
> > > > +/* Forward declarations */
> > > > +struct rte_auxiliary_driver;
> > > > +struct rte_auxiliary_bus;
> > > > +struct rte_auxiliary_device;
> > > > +
> > > > +/**
> > > > + * Match function for the driver to decide if device can be handled.
> > > > + */
> > > > +typedef bool(auxiliary_match_t) (const char *);
> > > > +
> > > > +/**
> > > > + * Initialization function for the driver called during auxiliary probing.
> > > > + */
> > > > +typedef int(auxiliary_probe_t) (struct rte_auxiliary_driver*,
> > > > +				struct rte_auxiliary_device*);
> > > > +
> > > > +/**
> > > > + * Uninitialization function for the driver called during hotplugging.
> > > > + */
> > > > +typedef int (auxiliary_remove_t)(struct rte_auxiliary_device *);
> > > > +
> > > > +/**
> > > > + * Driver-specific DMA mapping. After a successful call the
> > > > +device
> > > > + * will be able to read/write from/to this segment.
> > > > + *
> > > > + * @param dev
> > > > + *   Pointer to the auxiliary device.
> > > > + * @param addr
> > > > + *   Starting virtual address of memory to be mapped.
> > > > + * @param iova
> > > > + *   Starting IOVA address of memory to be mapped.
> > > > + * @param len
> > > > + *   Length of memory segment being mapped.
> > > > + * @return
> > > > + *   - 0 On success.
> > > > + *   - Negative value and rte_errno is set otherwise.
> > > > + */
> > > > +typedef int (auxiliary_dma_map_t)(struct rte_auxiliary_device *dev, void *addr,
> > > > +				  uint64_t iova, size_t len);
> > > > +
> > > > +/**
> > > > + * Driver-specific DMA un-mapping. After a successful call the
> > > > +device
> > > > + * will not be able to read/write from/to this segment.
> > > > + *
> > > > + * @param dev
> > > > + *   Pointer to the auxiliary device.
> > > > + * @param addr
> > > > + *   Starting virtual address of memory to be unmapped.
> > > > + * @param iova
> > > > + *   Starting IOVA address of memory to be unmapped.
> > > > + * @param len
> > > > + *   Length of memory segment being unmapped.
> > > > + * @return
> > > > + *   - 0 On success.
> > > > + *   - Negative value and rte_errno is set otherwise.
> > > > + */
> > > > +typedef int (auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > > +				    void *addr, uint64_t iova, size_t len);
> > > > +
> > > > +/**
> > > > + * A structure describing an auxiliary device.
> > > > + */
> > > > +struct rte_auxiliary_device {
> > > > +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> > > > +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> > > > +	struct rte_device device;                 /**< Inherit core device */
> > > > +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> > > > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> > > > +};
> > > > +
> > > > +/** List of auxiliary devices */
> > > > +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> > > > +/** List of auxiliary drivers */
> > > > +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> > > > +
> > > > +/**
> > > > + * Structure describing the auxiliary bus  */ struct
> > > > +rte_auxiliary_bus {
> > > > +	struct rte_bus bus;                  /**< Inherit the generic class */
> > > > +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> > > > +	struct rte_auxiliary_driver_list driver_list;  /**< List of
> > > > +drivers */ };
> > > > +
> > > > +/**
> > > > + * A structure describing an auxiliary driver.
> > > > + */
> > > > +struct rte_auxiliary_driver {
> > > > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > > > +	struct rte_driver driver;            /**< Inherit core driver. */
> > > > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > > > +	auxiliary_match_t *match;            /**< Device match function. */
> > > > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > > > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > > > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > > > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> > >
> > > These API type can be pointer type defined, then no need "*":
> > >
> > > typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > 				    void *addr, uint64_t iova, size_t len);
> > >
> > > auxiliary_dma_unmap_t dma_unmap;
> > >
> > > Like:
> > > https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759
> > > -1-thomas@monjalon.net/
> > >
> > > typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> > > 				 void *addr, uint64_t iova, size_t len);
> >
> > Thanks, is there a reason to prefer pointer type?
> 
> Good practice to make code beautiful ? ;-)
> 
> >
> > Thoma's patch looks good, will rebase on it once accepted.
> 
> I mean the function type is defined as pointer type.
Yes, I'm talking another topic :)
His patch makes dma map/unmap higher level to rte_bus, so that no need to define the api again here.

> 
> >
> > >
> > > > --
> > > > 2.25.1

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

15/04/2021 09:55, Xueming(Steven) Li:
> From: Wang, Haiyue <haiyue.wang@intel.com>
> > From: Xueming(Steven) Li <xuemingl@nvidia.com>
> > > From: Wang, Haiyue <haiyue.wang@intel.com>
> > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > > > +/**
> > > > > + * A structure describing an auxiliary driver.
> > > > > + */
> > > > > +struct rte_auxiliary_driver {
> > > > > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > > > > +	struct rte_driver driver;            /**< Inherit core driver. */
> > > > > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > > > > +	auxiliary_match_t *match;            /**< Device match function. */
> > > > > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > > > > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > > > > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > > > > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> > > >
> > > > These API type can be pointer type defined, then no need "*":
> > > >
> > > > typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > > 				    void *addr, uint64_t iova, size_t len);
> > > >
> > > > auxiliary_dma_unmap_t dma_unmap;
> > > >
> > > > Like:
> > > > https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759
> > > > -1-thomas@monjalon.net/
> > > >
> > > > typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> > > > 				 void *addr, uint64_t iova, size_t len);
> > >
> > > Thanks, is there a reason to prefer pointer type?
> > 
> > Good practice to make code beautiful ? ;-)

Honestly, I am not sure which one is better,
having the pointer type hidden in the typedef or explicit?

> > > Thoma's patch looks good, will rebase on it once accepted.
> > 
> > I mean the function type is defined as pointer type.
> Yes, I'm talking another topic :)
> His patch makes dma map/unmap higher level to rte_bus, so that no need to define the api again here.

I think my patch will be abandoned.

Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus

[Wang, Haiyue]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Thursday, April 15, 2021 15:59
> To: Wang, Haiyue <haiyue.wang@intel.com>; Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Asaf Penso <asafp@nvidia.com>; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; david.marchand@redhat.com; Yigit, Ferruh <ferruh.yigit@intel.com>;
> andrew.rybchenko@oktetlabs.ru; olivier.matz@6wind.com
> Subject: Re: [dpdk-dev] [PATCH v1] bus/auxiliary: introduce auxiliary bus
> 
> 15/04/2021 09:55, Xueming(Steven) Li:
> > From: Wang, Haiyue <haiyue.wang@intel.com>
> > > From: Xueming(Steven) Li <xuemingl@nvidia.com>
> > > > From: Wang, Haiyue <haiyue.wang@intel.com>
> > > > > From: dev <dev-bounces@dpdk.org> On Behalf Of Xueming Li
> > > > > > +/**
> > > > > > + * A structure describing an auxiliary driver.
> > > > > > + */
> > > > > > +struct rte_auxiliary_driver {
> > > > > > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > > > > > +	struct rte_driver driver;            /**< Inherit core driver. */
> > > > > > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > > > > > +	auxiliary_match_t *match;            /**< Device match function. */
> > > > > > +	auxiliary_probe_t *probe;            /**< Device Probe function. */
> > > > > > +	auxiliary_remove_t *remove;          /**< Device Remove function. */
> > > > > > +	auxiliary_dma_map_t *dma_map;        /**< Device dma map function. */
> > > > > > +	auxiliary_dma_unmap_t *dma_unmap;    /**< Device dma unmap function. */
> > > > >
> > > > > These API type can be pointer type defined, then no need "*":
> > > > >
> > > > > typedef int (*auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > > > > 				    void *addr, uint64_t iova, size_t len);
> > > > >
> > > > > auxiliary_dma_unmap_t dma_unmap;
> > > > >
> > > > > Like:
> > > > > https://patchwork.dpdk.org/project/dpdk/patch/20210331224547.2217759
> > > > > -1-thomas@monjalon.net/
> > > > >
> > > > > typedef int (*rte_dev_dma_map_t)(struct rte_device *dev,
> > > > > 				 void *addr, uint64_t iova, size_t len);
> > > >
> > > > Thanks, is there a reason to prefer pointer type?
> > >
> > > Good practice to make code beautiful ? ;-)
> 
> Honestly, I am not sure which one is better,
> having the pointer type hidden in the typedef or explicit?

I got this style from 'struct eth_dev_ops', Steven can make the final decision,
just a suggestion.

> 
> > > > Thoma's patch looks good, will rebase on it once accepted.
> > >
> > > I mean the function type is defined as pointer type.
> > Yes, I'm talking another topic :)
> > His patch makes dma map/unmap higher level to rte_bus, so that no need to define the api again here.
> 
> I think my patch will be abandoned.
> 

[dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary_device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
---
 MAINTAINERS                               |   5 +
 drivers/bus/auxiliary/auxiliary_common.c  | 408 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
 drivers/bus/auxiliary/meson.build         |  11 +
 drivers/bus/auxiliary/private.h           | 120 +++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 199 +++++++++++
 drivers/bus/auxiliary/version.map         |  10 +
 drivers/bus/meson.build                   |   1 +
 9 files changed, 959 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 44f3d322ed..ccab5b09c8 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -522,6 +522,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..cef85ae991
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,408 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+
+int auxiliary_bus_logtype;
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary Driver and Device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	/* no initialization when marked as blocked, return without error */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+		return -EINVAL;
+	}
+
+	dev->driver = dr;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+		      dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
+		      dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	const char **out = addr;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (dr->match(name))
+			break;
+	}
+	if (dr != NULL && addr != NULL)
+		*out = name;
+	return dr != NULL ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map)
+		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap)
+		return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..5a1b029839
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..b75bb4d4a6
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/**
+ * @file
+ * Linux auxiliary probing.
+ */
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?\n",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..f85608afd0
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+headers = files('rte_bus_auxiliary.h')
+sources = files('auxiliary_common.c',
+	'auxiliary_params.c')
+if is_linux
+	sources += files('linux/auxiliary.c')
+endif
+deps += ['kvargs']
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..3529348900
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+/**
+ * Test whether the auxiliary device exist
+ *
+ * @param name
+ *  Auxiliary device name
+ * @return
+ *  true on exists, false otherwise
+ */
+bool auxiliary_dev_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Setup or update device when being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param aux_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_aux_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_aux_dev
+ *	AUXILIARY device to be added before exist_aux_dev
+ * @return void
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param aux_drv
+ *      auxiliary driver
+ * @param aux_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..d681464602
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _RTE_BUS_AUXILIARY_H_
+#define _RTE_BUS_AUXILIARY_H_
+
+/**
+ * @file
+ *
+ * RTE Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(*rte_auxiliary_match_t) (const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(*rte_auxiliary_probe_t) (struct rte_auxiliary_driver *drv,
+				     struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (*rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (*rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (*rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t match;         /**< Device match function. */
+	rte_auxiliary_probe_t probe;         /**< Device Probe function. */
+	rte_auxiliary_remove_t remove;       /**< Device Remove function. */
+	rte_auxiliary_dma_map_t dma_map;     /**< Device dma map function. */
+	rte_auxiliary_dma_unmap_t dma_unmap; /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&auxiliary_drv);		\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BUS_AUXILIARY_H_ */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..3d270baea7
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,10 @@
+DPDK_21 {
+	local: *;
+};
+
+EXPERIMENTAL {
+	global:
+
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index c770c6ba1e..791536e606 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1

Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus

[Kinsella, Ray]

On 10/05/2021 14:47, Xueming Li wrote:
> Auxiliary [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary_device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> Cc: Wang Haiyue <haiyue.wang@intel.com>
> ---
>  MAINTAINERS                               |   5 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 408 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 147 ++++++++
>  drivers/bus/auxiliary/meson.build         |  11 +
>  drivers/bus/auxiliary/private.h           | 120 +++++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 199 +++++++++++
>  drivers/bus/auxiliary/version.map         |  10 +
>  drivers/bus/meson.build                   |   1 +
>  9 files changed, 959 insertions(+)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 44f3d322ed..ccab5b09c8 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -522,6 +522,11 @@ F: doc/guides/mempool/octeontx2.rst
>  Bus Drivers
>  -----------
>  
> +Auxiliary bus driver
> +M: Parav Pandit <parav@nvidia.com>
> +M: Xueming Li <xuemingl@nvidia.com>
> +F: drivers/bus/auxiliary/
> +
>  Intel FPGA bus
>  M: Rosen Xu <rosen.xu@intel.com>
>  F: drivers/bus/ifpga/
> diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
> new file mode 100644
> index 0000000000..cef85ae991
> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
> @@ -0,0 +1,408 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdbool.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <rte_errno.h>
> +#include <rte_interrupts.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_eal.h>
> +#include <rte_eal_paging.h>
> +#include <rte_string_fns.h>
> +#include <rte_common.h>
> +#include <rte_devargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +
> +int auxiliary_bus_logtype;
> +
> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name)
> +{
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
> +		if (strcmp(devargs->name, name) == 0)
> +			return devargs;
> +	}
> +	return NULL;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist
> + */
> +__rte_weak bool
> +auxiliary_dev_exists(const char *name)
> +{
> +	RTE_SET_USED(name);
> +	return false;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + */
> +__rte_weak int
> +auxiliary_scan(void)
> +{
> +	return 0;
> +}
> +
> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
> +{
> +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> +}
> +
> +/*
> + * Match the auxiliary Driver and Device using driver function.
> + */
> +bool
> +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		const struct rte_auxiliary_device *aux_dev)
> +{
> +	if (aux_drv->match == NULL)
> +		return false;
> +	return aux_drv->match(aux_dev->name);
> +}
> +
> +/*
> + * Call the probe() function of the driver.
> + */
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> +			       struct rte_auxiliary_device *dev)
> +{
> +	enum rte_iova_mode iova_mode;
> +	int ret;
> +
> +	if ((dr == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	/* The device is not blocked; Check if driver supports it */
> +	if (!auxiliary_match(dr, dev))
> +		/* Match of device and driver failed */
> +		return 1;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	/* no initialization when marked as blocked, return without error */
> +	if (dev->device.devargs != NULL &&
> +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> +		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
> +		return -1;
> +	}
> +
> +	if (dev->device.numa_node < 0) {
> +		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
> +		dev->device.numa_node = 0;
> +	}
> +
> +	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
> +
> +	iova_mode = rte_eal_iova_mode();
> +	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) &&
> +	    iova_mode != RTE_IOVA_VA) {
> +		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
> +		return -EINVAL;
> +	}
> +
> +	dev->driver = dr;
> +
> +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
> +		      dr->driver.name, dev->name, dev->device.numa_node);
> +	ret = dr->probe(dr, dev);
> +	if (ret)
> +		dev->driver = NULL;
> +	else
> +		dev->device.driver = &dr->driver;
> +
> +	return ret;
> +}
> +
> +/*
> + * Call the remove() function of the driver.
> + */
> +static int
> +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr;
> +	int ret = 0;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	dr = dev->driver;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
> +		      dev->name, dr->driver.name);
> +
> +	if (dr->remove) {
> +		ret = dr->remove(dev);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	/* clear driver structure */
> +	dev->driver = NULL;
> +	dev->device.driver = NULL;
> +
> +	return 0;
> +}
> +
> +/*
> + * Call the probe() function of all registered driver for the given device.
> + * Return < 0 if initialization failed.
> + * Return 1 if no driver is found for this device.
> + */
> +static int
> +auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr;
> +	int rc;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
> +		if (!dr->match(dev->name))
> +			continue;
> +
> +		rc = rte_auxiliary_probe_one_driver(dr, dev);
> +		if (rc < 0)
> +			/* negative value is an error */
> +			return rc;
> +		if (rc > 0)
> +			/* positive value means driver doesn't support it */
> +			continue;
> +		return 0;
> +	}
> +	return 1;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and call the probe() function for
> + *
> + * all registered drivers that have a matching entry in its id_table
> + * for discovered devices.
> + */
> +static int
> +auxiliary_probe(void)
> +{
> +	struct rte_auxiliary_device *dev = NULL;
> +	size_t probed = 0, failed = 0;
> +	int ret = 0;
> +
> +	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
> +		probed++;
> +
> +		ret = auxiliary_probe_all_drivers(dev);
> +		if (ret < 0) {
> +			if (ret != -EEXIST) {
> +				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
> +					      dev->name);
> +				rte_errno = errno;
> +				failed++;
> +			}
> +			ret = 0;
> +		}
> +	}
> +
> +	return (probed && probed == failed) ? -1 : 0;
> +}
> +
> +static int
> +auxiliary_parse(const char *name, void *addr)
> +{
> +	struct rte_auxiliary_driver *dr = NULL;
> +	const char **out = addr;
> +
> +	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
> +		if (dr->match(name))
> +			break;
> +	}
> +	if (dr != NULL && addr != NULL)
> +		*out = name;
> +	return dr != NULL ? 0 : -1;
> +}
> +
> +/* register a driver */
> +void
> +rte_auxiliary_register(struct rte_auxiliary_driver *driver)
> +{
> +	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
> +	driver->bus = &auxiliary_bus;
> +}
> +
> +/* unregister a driver */
> +void
> +rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
> +{
> +	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
> +	driver->bus = NULL;
> +}
> +
> +/* Add a device to auxiliary bus */
> +void
> +auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
> +{
> +	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
> +}
> +
> +/* Insert a device into a predefined position in auxiliary bus */
> +void
> +auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			struct rte_auxiliary_device *new_aux_dev)
> +{
> +	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
> +}
> +
> +/* Remove a device from auxiliary bus */
> +static void
> +rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
> +{
> +	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
> +}
> +
> +static struct rte_device *
> +auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> +		      const void *data)
> +{
> +	const struct rte_auxiliary_device *pstart;
> +	struct rte_auxiliary_device *adev;
> +
> +	if (start != NULL) {
> +		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
> +		adev = TAILQ_NEXT(pstart, next);
> +	} else {
> +		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
> +	}
> +	while (adev != NULL) {
> +		if (cmp(&adev->device, data) == 0)
> +			return &adev->device;
> +		adev = TAILQ_NEXT(adev, next);
> +	}
> +	return NULL;
> +}
> +
> +static int
> +auxiliary_plug(struct rte_device *dev)
> +{
> +	if (!auxiliary_dev_exists(dev->name))
> +		return -ENOENT;
> +	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
> +}
> +
> +static int
> +auxiliary_unplug(struct rte_device *dev)
> +{
> +	struct rte_auxiliary_device *adev;
> +	int ret;
> +
> +	adev = RTE_DEV_TO_AUXILIARY(dev);
> +	ret = rte_auxiliary_driver_remove_dev(adev);
> +	if (ret == 0) {
> +		rte_auxiliary_remove_device(adev);
> +		rte_devargs_remove(dev->devargs);
> +		free(adev);
> +	}
> +	return ret;
> +}
> +
> +static int
> +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> +{
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || !aux_dev->driver) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_map)
> +		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;
> +}
> +
> +static int
> +auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
> +		    size_t len)
> +{
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || !aux_dev->driver) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_unmap)
> +		return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;
> +}
> +
> +bool
> +auxiliary_ignore_device(const char *name)
> +{
> +	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
> +
> +	switch (auxiliary_bus.bus.conf.scan_mode) {
> +	case RTE_BUS_SCAN_ALLOWLIST:
> +		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
> +			return false;
> +		break;
> +	case RTE_BUS_SCAN_UNDEFINED:
> +	case RTE_BUS_SCAN_BLOCKLIST:
> +		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
> +			return false;
> +		break;
> +	}
> +	return true;
> +}
> +
> +static enum rte_iova_mode
> +auxiliary_get_iommu_class(void)
> +{
> +	const struct rte_auxiliary_driver *drv;
> +
> +	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
> +		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
> +			return RTE_IOVA_VA;
> +	}
> +
> +	return RTE_IOVA_DC;
> +}
> +
> +struct rte_auxiliary_bus auxiliary_bus = {
> +	.bus = {
> +		.scan = auxiliary_scan,
> +		.probe = auxiliary_probe,
> +		.find_device = auxiliary_find_device,
> +		.plug = auxiliary_plug,
> +		.unplug = auxiliary_unplug,
> +		.parse = auxiliary_parse,
> +		.dma_map = auxiliary_dma_map,
> +		.dma_unmap = auxiliary_dma_unmap,
> +		.get_iommu_class = auxiliary_get_iommu_class,
> +		.dev_iterate = auxiliary_dev_iterate,
> +	},
> +	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
> +	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
> +};
> +
> +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> +RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);
> diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
> new file mode 100644
> index 0000000000..5a1b029839
> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_params.c
> @@ -0,0 +1,58 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +
> +#include <rte_bus.h>
> +#include <rte_dev.h>
> +#include <rte_errno.h>
> +#include <rte_kvargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +enum auxiliary_params {
> +	RTE_AUXILIARY_PARAM_NAME,
> +};
> +
> +static const char * const auxiliary_params_keys[] = {
> +	[RTE_AUXILIARY_PARAM_NAME] = "name",
> +};
> +
> +static int
> +auxiliary_dev_match(const struct rte_device *dev,
> +	      const void *_kvlist)
> +{
> +	const struct rte_kvargs *kvlist = _kvlist;
> +	int ret;
> +
> +	ret = rte_kvargs_process(kvlist,
> +			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
> +			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
> +
> +	return ret != 0 ? -1 : 0;
> +}
> +
> +void *
> +auxiliary_dev_iterate(const void *start,
> +		    const char *str,
> +		    const struct rte_dev_iterator *it __rte_unused)
> +{
> +	rte_bus_find_device_t find_device;
> +	struct rte_kvargs *kvargs = NULL;
> +	struct rte_device *dev;
> +
> +	if (str != NULL) {
> +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> +		if (kvargs == NULL) {
> +			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
> +			rte_errno = EINVAL;
> +			return NULL;
> +		}
> +	}
> +	find_device = auxiliary_bus.bus.find_device;
> +	dev = find_device(start, auxiliary_dev_match, kvargs);
> +	rte_kvargs_free(kvargs);
> +	return dev;
> +}
> diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
> new file mode 100644
> index 0000000000..b75bb4d4a6
> --- /dev/null
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> @@ -0,0 +1,147 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#include <string.h>
> +#include <dirent.h>
> +
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <eal_filesystem.h>
> +
> +#include "../rte_bus_auxiliary.h"
> +#include "../private.h"
> +
> +#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
> +
> +/**
> + * @file
> + * Linux auxiliary probing.
> + */
> +
> +/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
> +static int
> +auxiliary_scan_one(const char *dirname, const char *name)
> +{
> +	struct rte_auxiliary_device *dev;
> +	struct rte_auxiliary_device *dev2;
> +	char filename[PATH_MAX];
> +	unsigned long tmp;
> +	int ret;
> +
> +	dev = malloc(sizeof(*dev));
> +	if (dev == NULL)
> +		return -1;
> +
> +	memset(dev, 0, sizeof(*dev));
> +	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
> +		free(dev);
> +		return -1;
> +	}
> +	dev->device.name = dev->name;
> +	dev->device.bus = &auxiliary_bus.bus;
> +
> +	/* Get numa node, default to 0 if not present */
> +	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
> +		 dirname, name);
> +	if (access(filename, F_OK) != -1) {
> +		if (eal_parse_sysfs_value(filename, &tmp) == 0)
> +			dev->device.numa_node = tmp;
> +		else
> +			dev->device.numa_node = -1;
> +	} else {
> +		dev->device.numa_node = 0;
> +	}
> +
> +	auxiliary_on_scan(dev);
> +
> +	/* Device is valid, add in list (sorted) */
> +	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
> +		ret = strcmp(dev->name, dev2->name);
> +		if (ret > 0)
> +			continue;
> +		if (ret < 0) {
> +			auxiliary_insert_device(dev2, dev);
> +		} else { /* already registered */
> +			if (rte_dev_is_probed(&dev2->device) &&
> +			    dev2->device.devargs != dev->device.devargs) {
> +				/* To probe device with new devargs. */
> +				rte_devargs_remove(dev2->device.devargs);
> +				auxiliary_on_scan(dev2);
> +			}
> +			free(dev);
> +		}
> +		return 0;
> +	}
> +	auxiliary_add_device(dev);
> +	return 0;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist
> + */
> +bool
> +auxiliary_dev_exists(const char *name)
> +{
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +
> +	snprintf(dirname, sizeof(dirname), "%s/%s",
> +		 AUXILIARY_SYSFS_PATH, name);
> +	dir = opendir(dirname);
> +	if (dir == NULL)
> +		return false;
> +	closedir(dir);
> +	return true;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + */
> +int
> +auxiliary_scan(void)
> +{
> +	struct dirent *e;
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +	struct rte_auxiliary_driver *drv;
> +
> +	dir = opendir(AUXILIARY_SYSFS_PATH);
> +	if (dir == NULL) {
> +		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?\n",
> +			      AUXILIARY_SYSFS_PATH);
> +		return 0;
> +	}
> +
> +	while ((e = readdir(dir)) != NULL) {
> +		if (e->d_name[0] == '.')
> +			continue;
> +
> +		if (auxiliary_ignore_device(e->d_name))
> +			continue;
> +
> +		snprintf(dirname, sizeof(dirname), "%s/%s",
> +			 AUXILIARY_SYSFS_PATH, e->d_name);
> +
> +		/* Ignore if no driver can handle. */
> +		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
> +			if (drv->match(e->d_name))
> +				break;
> +		}
> +		if (drv == NULL)
> +			continue;
> +
> +		if (auxiliary_scan_one(dirname, e->d_name) < 0)
> +			goto error;
> +	}
> +	closedir(dir);
> +	return 0;
> +
> +error:
> +	closedir(dir);
> +	return -1;
> +}
> diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
> new file mode 100644
> index 0000000000..f85608afd0
> --- /dev/null
> +++ b/drivers/bus/auxiliary/meson.build
> @@ -0,0 +1,11 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2021 Mellanox Technologies, Ltd
> +
> +headers = files('rte_bus_auxiliary.h')
> +sources = files('auxiliary_common.c',
> +	'auxiliary_params.c')
> +if is_linux
> +	sources += files('linux/auxiliary.c')
> +endif
> +deps += ['kvargs']
> +
> diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
> new file mode 100644
> index 0000000000..3529348900
> --- /dev/null
> +++ b/drivers/bus/auxiliary/private.h
> @@ -0,0 +1,120 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef _AUXILIARY_PRIVATE_H_
> +#define _AUXILIARY_PRIVATE_H_
> +
> +#include <stdbool.h>
> +#include <stdio.h>
> +#include "rte_bus_auxiliary.h"
> +
> +extern struct rte_auxiliary_bus auxiliary_bus;
> +extern int auxiliary_bus_logtype;
> +
> +#define AUXILIARY_LOG(level, fmt, args...) \
> +	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
> +		__func__, ##args)
> +
> +/* Auxiliary bus iterators */
> +#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
> +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> +
> +#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
> +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> +
> +/**
> + * Test whether the auxiliary device exist
> + *
> + * @param name
> + *  Auxiliary device name
> + * @return
> + *  true on exists, false otherwise
> + */
> +bool auxiliary_dev_exists(const char *name);
> +
> +/**
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + *
> + * @return
> + *  0 on success, negative on error
> + */
> +int auxiliary_scan(void);
> +
> +/**
> + * Setup or update device when being scanned.
> + *
> + * @param aux_dev
> + *	AUXILIARY device.
> + */
> +void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
> +
> +/**
> + * Validate whether a device with given auxiliary device should be ignored
> + * or not.
> + *
> + * @param name
> + *	Auxiliary name of device to be validated
> + * @return
> + *	true: if device is to be ignored,
> + *	false: if device is to be scanned,
> + */
> +bool auxiliary_ignore_device(const char *name);
> +
> +/**
> + * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
> + * list). This function also updates the bus references of the auxiliary
> + * Device (and the generic device object embedded within.
> + *
> + * @param aux_dev
> + *	AUXILIARY device to add
> + * @return void
> + */
> +void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
> +
> +/**
> + * Insert an auxiliary device in the auxiliary bus at a particular location
> + * in the device list. It also updates the auxiliary bus reference of the
> + * new devices to be inserted.
> + *
> + * @param exist_aux_dev
> + *	Existing auxiliary device in auxiliary bus
> + * @param new_aux_dev
> + *	AUXILIARY device to be added before exist_aux_dev
> + * @return void
> + */
> +void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			     struct rte_auxiliary_device *new_aux_dev);
> +
> +/**
> + * Match the auxiliary Driver and Device by driver function
> + *
> + * @param aux_drv
> + *      auxiliary driver
> + * @param aux_dev
> + *      auxiliary device to match against the driver
> + * @return
> + *      the driver can handle the device
> + */
> +bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		     const struct rte_auxiliary_device *aux_dev);
> +
> +/**
> + * Iterate over internal devices, matching any device against the provided
> + * string.
> + *
> + * @param start
> + *   Iteration starting point.
> + * @param str
> + *   Device string to match against.
> + * @param it
> + *   (unused) iterator structure.
> + * @return
> + *   A pointer to the next matching device if any.
> + *   NULL otherwise.
> + */
> +void *auxiliary_dev_iterate(const void *start, const char *str,
> +			    const struct rte_dev_iterator *it);
> +
> +#endif /* _AUXILIARY_PRIVATE_H_ */
> diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> new file mode 100644
> index 0000000000..d681464602
> --- /dev/null
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> @@ -0,0 +1,199 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef _RTE_BUS_AUXILIARY_H_
> +#define _RTE_BUS_AUXILIARY_H_
> +
> +/**
> + * @file
> + *
> + * RTE Auxiliary Bus Interface.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <sys/queue.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +
> +#include <rte_debug.h>
> +#include <rte_interrupts.h>
> +#include <rte_dev.h>
> +#include <rte_bus.h>
> +#include <rte_kvargs.h>
> +
> +/* Forward declarations */
> +struct rte_auxiliary_driver;
> +struct rte_auxiliary_bus;
> +struct rte_auxiliary_device;
> +
> +/**
> + * Match function for the driver to decide if device can be handled.
> + *
> + * @param name
> + *   Pointer to the auxiliary device name.
> + * @return
> + *   Whether the driver can handle the auxiliary device.
> + */
> +typedef bool(*rte_auxiliary_match_t) (const char *name);
> +
> +/**
> + * Initialization function for the driver called during auxiliary probing.
> + *
> + * @param drv
> + *   Pointer to the auxiliary driver.
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int(*rte_auxiliary_probe_t) (struct rte_auxiliary_driver *drv,
> +				     struct rte_auxiliary_device *dev);
> +
> +/**
> + * Uninitialization function for the driver called during hotplugging.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (*rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
> +
> +/**
> + * Driver-specific DMA mapping. After a successful call the device
> + * will be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be mapped.
> + * @param iova
> + *   Starting IOVA address of memory to be mapped.
> + * @param len
> + *   Length of memory segment being mapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (*rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
> +				       void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * Driver-specific DMA un-mapping. After a successful call the device
> + * will not be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be unmapped.
> + * @param iova
> + *   Starting IOVA address of memory to be unmapped.
> + * @param len
> + *   Length of memory segment being unmapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (*rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> +					 void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * A structure describing an auxiliary device.
> + */
> +struct rte_auxiliary_device {
> +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> +	struct rte_device device;                 /**< Inherit core device */
> +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> +};
> +
> +/** List of auxiliary devices */
> +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> +/** List of auxiliary drivers */
> +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> +
> +/**
> + * Structure describing the auxiliary bus
> + */
> +struct rte_auxiliary_bus {
> +	struct rte_bus bus;                  /**< Inherit the generic class */
> +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
> +};
> +
> +/**
> + * A structure describing an auxiliary driver.
> + */
> +struct rte_auxiliary_driver {
> +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> +	struct rte_driver driver;            /**< Inherit core driver. */
> +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> +	rte_auxiliary_match_t match;         /**< Device match function. */
> +	rte_auxiliary_probe_t probe;         /**< Device Probe function. */
> +	rte_auxiliary_remove_t remove;       /**< Device Remove function. */
> +	rte_auxiliary_dma_map_t dma_map;     /**< Device dma map function. */
> +	rte_auxiliary_dma_unmap_t dma_unmap; /**< Device dma unmap function. */
> +	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
> +};
> +
> +/**
> + * @internal
> + * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
> + */
> +#define RTE_DEV_TO_AUXILIARY(ptr) \
> +	container_of(ptr, struct rte_auxiliary_device, device)
> +
> +#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
> +	container_of(ptr, const struct rte_auxiliary_device, device)
> +
> +#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
> +	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
> +
> +/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
> +#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
> +
> +/**
> + * Register an auxiliary driver.
> + *
> + * @param driver
> + *   A pointer to a rte_auxiliary_driver structure describing the driver
> + *   to be registered.
> + */
> +__rte_experimental
> +void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
> +
> +/** Helper for auxiliary device registration from driver instance */
> +#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
> +	RTE_INIT(auxiliaryinitfn_##nm)				\
> +	{							\
> +		(auxiliary_drv).driver.name = RTE_STR(nm);	\
> +		rte_auxiliary_register(&auxiliary_drv);		\
> +	}							\
> +	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
> +
> +/**
> + * Unregister an auxiliary driver.
> + *
> + * @param driver
> + *   A pointer to a rte_auxiliary_driver structure describing the driver
> + *   to be unregistered.
> + */
> +__rte_experimental
> +void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* _RTE_BUS_AUXILIARY_H_ */
> diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
> new file mode 100644
> index 0000000000..3d270baea7
> --- /dev/null
> +++ b/drivers/bus/auxiliary/version.map
> @@ -0,0 +1,10 @@
> +DPDK_21 {
> +	local: *;
> +};
> +
> +EXPERIMENTAL {
> +	global:
> +

Add a comment here to indicate the release in which the expiermental symbols where added. 

> +	rte_auxiliary_register;
> +	rte_auxiliary_unregister;
> +};
> diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
> index c770c6ba1e..791536e606 100644
> --- a/drivers/bus/meson.build
> +++ b/drivers/bus/meson.build
> @@ -2,6 +2,7 @@
>  # Copyright(c) 2017 Intel Corporation
>  
>  drivers = [
> +        'auxiliary',
>          'dpaa',
>          'fslmc',
>          'ifpga',
> 

Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

10/05/2021 15:47, Xueming Li:
> Auxiliary [1] provides a way to split function into child-devices

Auxiliary -> Auxiliary bus

> representing sub-domains of functionality. Each auxiliary_device

auxiliary_device -> auxiliary device

> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
[...]
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
[...]
> +int auxiliary_bus_logtype;

You don't need to declare this variable, it is done in a macro.

> +
> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name)
> +{
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {

"auxiliary" should be defined as a macro.

[...]
> +/*
> + * Test whether the auxiliary device exist
> + */
> +__rte_weak bool

The comment should explain it is a stub for OS
not supporting auxiliary bus.

> +auxiliary_dev_exists(const char *name)
> +{
> +	RTE_SET_USED(name);
> +	return false;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + */

Same here about the stub comment.

> +__rte_weak int
> +auxiliary_scan(void)
> +{
> +	return 0;
> +}
> +

Please insert a comment here.

> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
> +{
> +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> +}
> +
> +/*
> + * Match the auxiliary Driver and Device using driver function.

driver and device do not need capital letters

> + */
> +bool
> +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		const struct rte_auxiliary_device *aux_dev)
> +{
> +	if (aux_drv->match == NULL)
> +		return false;
> +	return aux_drv->match(aux_dev->name);
> +}
> +
> +/*
> + * Call the probe() function of the driver.
> + */
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> +			       struct rte_auxiliary_device *dev)
> +{
> +	enum rte_iova_mode iova_mode;
> +	int ret;
> +
> +	if ((dr == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	/* The device is not blocked; Check if driver supports it */

Please keep all the comments more uniform by starting with a capital
and ends with a point.

[...]
> +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> +RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);

Please replace with RTE_LOG_REGISTER_DEFAULT.

[...]
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
[...]
> +/**
> + * @file
> + * Linux auxiliary probing.
> + */

No need of doxygen comment in a .c file.

[...]
> +++ b/drivers/bus/auxiliary/meson.build
> @@ -0,0 +1,11 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2021 Mellanox Technologies, Ltd
> +
> +headers = files('rte_bus_auxiliary.h')
> +sources = files('auxiliary_common.c',
> +	'auxiliary_params.c')
> +if is_linux
> +	sources += files('linux/auxiliary.c')
> +endif
> +deps += ['kvargs']

Please change the indent with spaces
and check with devtools/check-meson.py

[...]
> +++ b/drivers/bus/auxiliary/private.h
[...]
> +/* Auxiliary bus iterators */
> +#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\

Please avoid tabs at the end of the line.
A space is enough before the backslash.

> +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> +
> +#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
> +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> +
> +/**
> + * Test whether the auxiliary device exist
> + *
> + * @param name
> + *  Auxiliary device name
> + * @return
> + *  true on exists, false otherwise
> + */
> +bool auxiliary_dev_exists(const char *name);

Given it is not an API, no need of doxygen.
And the function is so simple that no need of comment at all.

[...]
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> +#ifndef _RTE_BUS_AUXILIARY_H_
> +#define _RTE_BUS_AUXILIARY_H_

No need of underscores before and after.
(yes I know a lot of DPDK files use such scheme)

> +
> +/**
> + * @file
> + *
> + * RTE Auxiliary Bus Interface.

No need of RTE, it has no meaning here.

> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <sys/queue.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +
> +#include <rte_debug.h>
> +#include <rte_interrupts.h>
> +#include <rte_dev.h>
> +#include <rte_bus.h>
> +#include <rte_kvargs.h>
> +
> +/* Forward declarations */
> +struct rte_auxiliary_driver;
> +struct rte_auxiliary_bus;
> +struct rte_auxiliary_device;
> +
> +/**
> + * Match function for the driver to decide if device can be handled.
> + *
> + * @param name
> + *   Pointer to the auxiliary device name.
> + * @return
> + *   Whether the driver can handle the auxiliary device.
> + */
> +typedef bool(*rte_auxiliary_match_t) (const char *name);

I disagree with the earlier comment asking for typedef pointer
(based on one of my patches).
Actually Andrew's suggestion makes sense:
	http://mails.dpdk.org/archives/dev/2021-June/210665.html

[...]
> +++ b/drivers/bus/auxiliary/version.map
> +DPDK_21 {
> +	local: *;
> +};

We don't need this empty section.

> +
> +EXPERIMENTAL {
> +	global:
> +

As asked by Ray, we should add this line:
	# added in 21.08

> +	rte_auxiliary_register;
> +	rte_auxiliary_unregister;
> +};

Release notes are missing.

Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus

[Wang, Haiyue]

Hi Andrew,

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, June 8, 2021 15:53
> To: Xueming Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Wang, Haiyue
> <haiyue.wang@intel.com>; andrew.rybchenko@oktetlabs.ru
> Subject: Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> 


> > +
> > +/**
> > + * Match function for the driver to decide if device can be handled.
> > + *
> > + * @param name
> > + *   Pointer to the auxiliary device name.
> > + * @return
> > + *   Whether the driver can handle the auxiliary device.
> > + */
> > +typedef bool(*rte_auxiliary_match_t) (const char *name);
> 
> I disagree with the earlier comment asking for typedef pointer
> (based on one of my patches).
> Actually Andrew's suggestion makes sense:
> 	http://mails.dpdk.org/archives/dev/2021-June/210665.html
> 

I didn't get the error, but the same warning, I missed something ? ;-)

1. w/o pointer

#include <stdio.h>

typedef int (gpu_malloc_t)(void *dev, size_t size, void **ptr);

static gpu_malloc_t *mlx5_gpu_malloc_fn;

static int
mlx5_gpu_malloc(size_t size, void **ptr)
{
	return 0;
}

int main()
{
	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;

	mlx5_gpu_malloc_fn(NULL, 0, NULL);

	return 0;
}


gcc -Wall fun.c
fun.c: In function 'main':
fun.c:15:21: warning: assignment to 'int (*)(void *, size_t,  void **)' {aka 'int (*)(void *, long unsigned int,  void **)'} from incompatible pointer type 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-types]
   15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
      |                     ^



2. w pointer

#include <stdio.h>

typedef int (*gpu_malloc_t)(void *dev, size_t size, void **ptr);

static gpu_malloc_t mlx5_gpu_malloc_fn;

static int
mlx5_gpu_malloc(size_t size, void **ptr)
{
	return 0;
}

int main()
{
	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;

	mlx5_gpu_malloc_fn(NULL, 0, NULL);

	return 0;
}

gcc -Wall fun.c
fun.c: In function 'main':
fun.c:15:21: warning: assignment to 'gpu_malloc_t' {aka 'int (*)(void *, long unsigned int,  void **)'} from incompatible pointer type 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-types]
   15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
      |                     ^

> 

Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Kinsella, Ray <mdr@ashroe.eu>
> Sent: Tuesday, May 11, 2021 5:48 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Neil Horman <nhorman@tuxdriver.com>; Wang Haiyue
> <haiyue.wang@intel.com>
> Subject: Re: [RFC v2] bus/auxiliary: introduce auxiliary bus
> 


// snip //

> > diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
> > new file mode 100644
> > index 0000000000..3d270baea7
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/version.map
> > @@ -0,0 +1,10 @@
> > +DPDK_21 {
> > +	local: *;
> > +};
> > +
> > +EXPERIMENTAL {
> > +	global:
> > +
> 
> Add a comment here to indicate the release in which the expiermental symbols where added.

Good suggestion, thanks!

> 
> > +	rte_auxiliary_register;
> > +	rte_auxiliary_unregister;
> > +};

// snip //

Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Wang, Haiyue <haiyue.wang@intel.com>
> Sent: Tuesday, June 8, 2021 4:41 PM
> To: NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; andrew.rybchenko@oktetlabs.ru
> Subject: RE: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> 
> Hi Andrew,
> 
> > -----Original Message-----
> > From: Thomas Monjalon <thomas@monjalon.net>
> > Sent: Tuesday, June 8, 2021 15:53
> > To: Xueming Li <xuemingl@nvidia.com>
> > Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> > <mdr@ashroe.eu>; Wang, Haiyue <haiyue.wang@intel.com>;
> > andrew.rybchenko@oktetlabs.ru
> > Subject: Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary
> > bus
> >
> 
> 
> > > +
> > > +/**
> > > + * Match function for the driver to decide if device can be handled.
> > > + *
> > > + * @param name
> > > + *   Pointer to the auxiliary device name.
> > > + * @return
> > > + *   Whether the driver can handle the auxiliary device.
> > > + */
> > > +typedef bool(*rte_auxiliary_match_t) (const char *name);
> >
> > I disagree with the earlier comment asking for typedef pointer (based
> > on one of my patches).
> > Actually Andrew's suggestion makes sense:
> > 	http://mails.dpdk.org/archives/dev/2021-June/210665.html
> >
> 
> I didn't get the error, but the same warning, I missed something ? ;-)
> 
> 1. w/o pointer
> 
> #include <stdio.h>
> 
> typedef int (gpu_malloc_t)(void *dev, size_t size, void **ptr);
> 
> static gpu_malloc_t *mlx5_gpu_malloc_fn;
> 
> static int
> mlx5_gpu_malloc(size_t size, void **ptr) {
> 	return 0;
> }
> 
> int main()
> {
> 	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> 
> 	mlx5_gpu_malloc_fn(NULL, 0, NULL);
> 
> 	return 0;
> }
> 
> 
> gcc -Wall fun.c
> fun.c: In function 'main':
> fun.c:15:21: warning: assignment to 'int (*)(void *, size_t,  void **)' {aka 'int (*)(void *, long unsigned int,  void **)'} from incompatible
> pointer type 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-types]
>    15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
>       |                     ^
> 
> 
> 
> 2. w pointer
> 
> #include <stdio.h>
> 
> typedef int (*gpu_malloc_t)(void *dev, size_t size, void **ptr);
> 
> static gpu_malloc_t mlx5_gpu_malloc_fn;
> 
> static int
> mlx5_gpu_malloc(size_t size, void **ptr) {
> 	return 0;
> }
> 
> int main()
> {
> 	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> 
> 	mlx5_gpu_malloc_fn(NULL, 0, NULL);
> 
> 	return 0;
> }
> 
> gcc -Wall fun.c
> fun.c: In function 'main':
> fun.c:15:21: warning: assignment to 'gpu_malloc_t' {aka 'int (*)(void *, long unsigned int,  void **)'} from incompatible pointer type
> 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-types]
>    15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
>       |                     ^
> 
> >

I don't mind as both suggestion works.
From usage perspective, it's straightforward for anybody to see an pointer field and know "that's a pointer to a something".

Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, June 8, 2021 3:53 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>; Wang Haiyue <haiyue.wang@intel.com>;
> andrew.rybchenko@oktetlabs.ru
> Subject: Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> 
> 10/05/2021 15:47, Xueming Li:
> > Auxiliary [1] provides a way to split function into child-devices
> 
> Auxiliary -> Auxiliary bus
> 
> > representing sub-domains of functionality. Each auxiliary_device
> 
> auxiliary_device -> auxiliary device
> 
> > represents a part of its parent functionality.
> >
> > Auxiliary device is identified by unique device name, sysfs path:
> >   /sys/bus/auxiliary/devices/<name>
> >
> > [1] kernel auxiliary bus document:
> > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> [...]
> > +++ b/drivers/bus/auxiliary/auxiliary_common.c
> [...]
> > +int auxiliary_bus_logtype;
> 
> You don't need to declare this variable, it is done in a macro.
> 
> > +
> > +static struct rte_devargs *
> > +auxiliary_devargs_lookup(const char *name) {
> > +	struct rte_devargs *devargs;
> > +
> > +	RTE_EAL_DEVARGS_FOREACH("auxiliary", devargs) {
> 
> "auxiliary" should be defined as a macro.
> 
> [...]
> > +/*
> > + * Test whether the auxiliary device exist  */ __rte_weak bool
> 
> The comment should explain it is a stub for OS not supporting auxiliary bus.
> 
> > +auxiliary_dev_exists(const char *name) {
> > +	RTE_SET_USED(name);
> > +	return false;
> > +}
> > +
> > +/*
> > + * Scan the content of the auxiliary bus, and the devices in the
> > +devices
> > + * list
> > + */
> 
> Same here about the stub comment.
> 
> > +__rte_weak int
> > +auxiliary_scan(void)
> > +{
> > +	return 0;
> > +}
> > +
> 
> Please insert a comment here.
> 
> > +void
> > +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev) {
> > +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> > +}
> > +
> > +/*
> > + * Match the auxiliary Driver and Device using driver function.
> 
> driver and device do not need capital letters
> 
> > + */
> > +bool
> > +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> > +		const struct rte_auxiliary_device *aux_dev) {
> > +	if (aux_drv->match == NULL)
> > +		return false;
> > +	return aux_drv->match(aux_dev->name); }
> > +
> > +/*
> > + * Call the probe() function of the driver.
> > + */
> > +static int
> > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > +			       struct rte_auxiliary_device *dev)
> > +{
> > +	enum rte_iova_mode iova_mode;
> > +	int ret;
> > +
> > +	if ((dr == NULL) || (dev == NULL))
> > +		return -EINVAL;
> > +
> > +	/* The device is not blocked; Check if driver supports it */
> 
> Please keep all the comments more uniform by starting with a capital
> and ends with a point.
> 
> [...]
> > +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> > +RTE_LOG_REGISTER(auxiliary_bus_logtype, bus.auxiliary, NOTICE);
> 
> Please replace with RTE_LOG_REGISTER_DEFAULT.
> 
> [...]
> > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> [...]
> > +/**
> > + * @file
> > + * Linux auxiliary probing.
> > + */
> 
> No need of doxygen comment in a .c file.
> 
> [...]
> > +++ b/drivers/bus/auxiliary/meson.build
> > @@ -0,0 +1,11 @@
> > +# SPDX-License-Identifier: BSD-3-Clause
> > +# Copyright 2021 Mellanox Technologies, Ltd
> > +
> > +headers = files('rte_bus_auxiliary.h')
> > +sources = files('auxiliary_common.c',
> > +	'auxiliary_params.c')
> > +if is_linux
> > +	sources += files('linux/auxiliary.c')
> > +endif
> > +deps += ['kvargs']
> 
> Please change the indent with spaces
> and check with devtools/check-meson.py
> 
> [...]
> > +++ b/drivers/bus/auxiliary/private.h
> [...]
> > +/* Auxiliary bus iterators */
> > +#define FOREACH_DEVICE_ON_AUXILIARYBUS(p)	\
> 
> Please avoid tabs at the end of the line.
> A space is enough before the backslash.
> 
> > +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> > +
> > +#define FOREACH_DRIVER_ON_AUXILIARYBUS(p)	\
> > +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> > +
> > +/**
> > + * Test whether the auxiliary device exist
> > + *
> > + * @param name
> > + *  Auxiliary device name
> > + * @return
> > + *  true on exists, false otherwise
> > + */
> > +bool auxiliary_dev_exists(const char *name);
> 
> Given it is not an API, no need of doxygen.
> And the function is so simple that no need of comment at all.
> 
> [...]
> > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > +#ifndef _RTE_BUS_AUXILIARY_H_
> > +#define _RTE_BUS_AUXILIARY_H_
> 
> No need of underscores before and after.
> (yes I know a lot of DPDK files use such scheme)
> 
> > +
> > +/**
> > + * @file
> > + *
> > + * RTE Auxiliary Bus Interface.
> 
> No need of RTE, it has no meaning here.
> 
> > + */
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <limits.h>
> > +#include <errno.h>
> > +#include <sys/queue.h>
> > +#include <stdint.h>
> > +#include <inttypes.h>
> > +
> > +#include <rte_debug.h>
> > +#include <rte_interrupts.h>
> > +#include <rte_dev.h>
> > +#include <rte_bus.h>
> > +#include <rte_kvargs.h>
> > +
> > +/* Forward declarations */
> > +struct rte_auxiliary_driver;
> > +struct rte_auxiliary_bus;
> > +struct rte_auxiliary_device;
> > +
> > +/**
> > + * Match function for the driver to decide if device can be handled.
> > + *
> > + * @param name
> > + *   Pointer to the auxiliary device name.
> > + * @return
> > + *   Whether the driver can handle the auxiliary device.
> > + */
> > +typedef bool(*rte_auxiliary_match_t) (const char *name);
> 
> I disagree with the earlier comment asking for typedef pointer
> (based on one of my patches).
> Actually Andrew's suggestion makes sense:
> 	http://mails.dpdk.org/archives/dev/2021-June/210665.html
> 
> [...]
> > +++ b/drivers/bus/auxiliary/version.map
> > +DPDK_21 {
> > +	local: *;
> > +};
> 
> We don't need this empty section.
> 
> > +
> > +EXPERIMENTAL {
> > +	global:
> > +
> 
> As asked by Ray, we should add this line:
> 	# added in 21.08
> 
> > +	rte_auxiliary_register;
> > +	rte_auxiliary_unregister;
> > +};
> 
> Release notes are missing.
> 

Thanks for all the good catches, will update and reflect in next version.

Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus

[Wang, Haiyue]

> -----Original Message-----
> From: Xueming(Steven) Li <xuemingl@nvidia.com>
> Sent: Thursday, June 10, 2021 14:30
> To: Wang, Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>;
> andrew.rybchenko@oktetlabs.ru
> Subject: RE: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> 
> 
> 
> > -----Original Message-----
> > From: Wang, Haiyue <haiyue.wang@intel.com>
> > Sent: Tuesday, June 8, 2021 4:41 PM
> > To: NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Xueming(Steven) Li <xuemingl@nvidia.com>
> > Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella <mdr@ashroe.eu>;
> andrew.rybchenko@oktetlabs.ru
> > Subject: RE: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary bus
> >
> > Hi Andrew,
> >
> > > -----Original Message-----
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > Sent: Tuesday, June 8, 2021 15:53
> > > To: Xueming Li <xuemingl@nvidia.com>
> > > Cc: dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> > > <mdr@ashroe.eu>; Wang, Haiyue <haiyue.wang@intel.com>;
> > > andrew.rybchenko@oktetlabs.ru
> > > Subject: Re: [dpdk-dev] [RFC v2] bus/auxiliary: introduce auxiliary
> > > bus
> > >
> >
> >
> > > > +
> > > > +/**
> > > > + * Match function for the driver to decide if device can be handled.
> > > > + *
> > > > + * @param name
> > > > + *   Pointer to the auxiliary device name.
> > > > + * @return
> > > > + *   Whether the driver can handle the auxiliary device.
> > > > + */
> > > > +typedef bool(*rte_auxiliary_match_t) (const char *name);
> > >
> > > I disagree with the earlier comment asking for typedef pointer (based
> > > on one of my patches).
> > > Actually Andrew's suggestion makes sense:
> > > 	http://mails.dpdk.org/archives/dev/2021-June/210665.html
> > >
> >
> > I didn't get the error, but the same warning, I missed something ? ;-)
> >
> > 1. w/o pointer
> >
> > #include <stdio.h>
> >
> > typedef int (gpu_malloc_t)(void *dev, size_t size, void **ptr);
> >
> > static gpu_malloc_t *mlx5_gpu_malloc_fn;
> >
> > static int
> > mlx5_gpu_malloc(size_t size, void **ptr) {
> > 	return 0;
> > }
> >
> > int main()
> > {
> > 	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> >
> > 	mlx5_gpu_malloc_fn(NULL, 0, NULL);
> >
> > 	return 0;
> > }
> >
> >
> > gcc -Wall fun.c
> > fun.c: In function 'main':
> > fun.c:15:21: warning: assignment to 'int (*)(void *, size_t,  void **)' {aka 'int (*)(void *, long
> unsigned int,  void **)'} from incompatible
> > pointer type 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-
> Wincompatible-pointer-types]
> >    15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> >       |                     ^
> >
> >
> >
> > 2. w pointer
> >
> > #include <stdio.h>
> >
> > typedef int (*gpu_malloc_t)(void *dev, size_t size, void **ptr);
> >
> > static gpu_malloc_t mlx5_gpu_malloc_fn;
> >
> > static int
> > mlx5_gpu_malloc(size_t size, void **ptr) {
> > 	return 0;
> > }
> >
> > int main()
> > {
> > 	mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> >
> > 	mlx5_gpu_malloc_fn(NULL, 0, NULL);
> >
> > 	return 0;
> > }
> >
> > gcc -Wall fun.c
> > fun.c: In function 'main':
> > fun.c:15:21: warning: assignment to 'gpu_malloc_t' {aka 'int (*)(void *, long unsigned int,  void
> **)'} from incompatible pointer type
> > 'int (*)(size_t,  void **)' {aka 'int (*)(long unsigned int,  void **)'} [-Wincompatible-pointer-
> types]
> >    15 |  mlx5_gpu_malloc_fn = mlx5_gpu_malloc;
> >       |                     ^
> >
> > >
> 
> I don't mind as both suggestion works.

I just want to understand what kind of error the compiler will complain about. ;-)

> From usage perspective, it's straightforward for anybody to see an pointer field and know "that's a
> pointer to a something".

> 

[dpdk-dev] [PATCH v3 1/2] devargs: add common key definition

[Xueming Li]

Adds common devargs key definition for "bus", "class" and "driver".

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |  2 --
 drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
 drivers/common/sfc_efx/sfc_efx.h        |  2 --
 drivers/net/bonding/rte_eth_bond_args.c |  2 +-
 drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
 drivers/net/iavf/iavf_ethdev.c          |  5 ++---
 drivers/net/mlx5/mlx5.c                 |  4 ++--
 drivers/net/sfc/sfc_kvargs.c            |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
 lib/eal/common/eal_common_devargs.c     | 12 ++++++------
 lib/eal/include/rte_devargs.h           |  4 ++++
 11 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..306f2f1ab7 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -208,8 +208,6 @@ __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
 
-#define MLX5_CLASS_ARG_NAME "class"
-
 enum mlx5_class {
 	MLX5_CLASS_INVALID,
 	MLX5_CLASS_NET = RTE_BIT64(0),
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..34747c4e07 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,
 static int
 parse_class_options(const struct rte_devargs *devargs)
 {
-	const char *key = MLX5_CLASS_ARG_NAME;
+	const char *key = RTE_DEVARGS_KEY_CLASS;
 	struct rte_kvargs *kvlist;
 	int ret = 0;
 
diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
index 6b6164cb1f..c16eca60f3 100644
--- a/drivers/common/sfc_efx/sfc_efx.h
+++ b/drivers/common/sfc_efx/sfc_efx.h
@@ -19,8 +19,6 @@
 extern "C" {
 #endif
 
-#define SFC_EFX_KVARG_DEV_CLASS	"class"
-
 enum sfc_efx_dev_class {
 	SFC_EFX_DEV_CLASS_INVALID = 0,
 	SFC_EFX_DEV_CLASS_NET,
diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
index 764b1b8c8e..5406e1c934 100644
--- a/drivers/net/bonding/rte_eth_bond_args.c
+++ b/drivers/net/bonding/rte_eth_bond_args.c
@@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
 	PMD_BOND_SOCKET_ID_KVARG,
 	PMD_BOND_MAC_ADDR_KVARG,
 	PMD_BOND_AGG_MODE_KVARG,
-	"driver",
+	RTE_DEVARGS_KEY_DRIVER,
 	NULL
 };
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index cb898bdb68..1d8ca42a0f 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1660,7 +1660,6 @@ static int
 i40evf_driver_selected(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (devargs == NULL)
@@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       i40evf_check_driver_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index d688c31cfb..4e79319017 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -2440,7 +2440,6 @@ static int
 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (device_id != IAVF_DEV_ID_VF &&
@@ -2456,13 +2455,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       iavf_drv_i40evf_check_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..d0faa45944 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->max_dump_files_num = tmp;
 	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
 		config->lro.timeout = tmp;
-	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
+	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
 		DRV_LOG(DEBUG, "class argument is %s.", val);
 	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
 		config->log_hp_size = tmp;
@@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_REPRESENTOR,
 		MLX5_MAX_DUMP_FILES_NUM,
 		MLX5_LRO_TIMEOUT_USEC,
-		MLX5_CLASS_ARG_NAME,
+		RTE_DEVARGS_KEY_CLASS,
 		MLX5_HP_BUF_SIZE,
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 0efa92ed28..974c05e68e 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
 		SFC_KVARG_TX_DATAPATH,
 		SFC_KVARG_FW_VARIANT,
 		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
-		SFC_EFX_KVARG_DEV_CLASS,
+		RTE_DEVARGS_KEY_CLASS,
 		NULL,
 	};
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index e5e03e6582..8b5bfd8c3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 	unsigned long tmp;
 	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 
-	if (strcmp(key, "class") == 0)
+	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
 		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
index b31ac879a9..23aaf8b7e4 100644
--- a/lib/eal/common/eal_common_devargs.c
+++ b/lib/eal/common/eal_common_devargs.c
@@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		const char *str;
 		struct rte_kvargs *kvlist;
 	} layers[] = {
-		{ "bus=",    NULL, NULL, },
-		{ "class=",  NULL, NULL, },
-		{ "driver=", NULL, NULL, },
+		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
+		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
+		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
 	};
 	struct rte_kvargs_pair *kv = NULL;
 	struct rte_class *cls = NULL;
@@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		if (layers[i].kvlist == NULL)
 			continue;
 		kv = &layers[i].kvlist->pairs[0];
-		if (strcmp(kv->key, "bus") == 0) {
+		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
 			bus = rte_bus_find_by_name(kv->value);
 			if (bus == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
@@ -126,7 +126,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "class") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
 			cls = rte_class_find_by_name(kv->value);
 			if (cls == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
@@ -134,7 +134,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "driver") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
 			/* Ignore */
 			continue;
 		}
diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h
index 1e595b3c51..14fe7f70b6 100644
--- a/lib/eal/include/rte_devargs.h
+++ b/lib/eal/include/rte_devargs.h
@@ -25,6 +25,10 @@ extern "C" {
 #include <rte_compat.h>
 #include <rte_bus.h>
 
+#define RTE_DEVARGS_KEY_BUS "bus"
+#define RTE_DEVARGS_KEY_CLASS "class"
+#define RTE_DEVARGS_KEY_DRIVER "driver"
+
 /**
  * Type of generic device
  */
-- 
2.25.1

[dpdk-dev] [PATCH v3 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary bus [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

Devargs syntax of auxiliary device:
  -a auxiliary:<name>[,args...]

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
Cc: Thomas Monjalon <thomas@monjalon.net>
Cc: Kinsella Ray <mdr@ashroe.eu>
---
 MAINTAINERS                               |   5 +
 doc/guides/rel_notes/release_21_08.rst    |   7 +
 drivers/bus/auxiliary/auxiliary_common.c  | 419 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 142 ++++++++
 drivers/bus/auxiliary/meson.build         |  11 +
 drivers/bus/auxiliary/private.h           | 112 ++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
 drivers/bus/auxiliary/version.map         |   7 +
 drivers/bus/meson.build                   |   1 +
 10 files changed, 963 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..eaf691ca6a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..b335064963 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,13 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added auxiliary bus support.**
+
+  * Auxiliary bus provides a way to split function into child-devices
+    representing sub-domains of functionality. Each auxiliary device
+    represents a part of its parent functionality.
+  * Devargs syntax of auxiliary device: -a auxiliary:<name>[,args...]
+
 
 Removed Items
 -------------
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..91008bdc80
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,419 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AXILIARY_NAME, devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+/**
+ * Update a device being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary driver and device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it. */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	/* No initialization when marked as blocked, return without error. */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+		return -EINVAL;
+	}
+
+	dev->driver = dr;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+		      dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
+		      dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	const char **out = addr;
+
+	/* Allow dummy name to prevent bus scan. */
+	if (strlen(name) == 0)
+		return 0;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (dr->match(name))
+			break;
+	}
+	if (dr != NULL && addr != NULL)
+		*out = name;
+	return dr != NULL ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map)
+		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap)
+		return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..5a1b029839
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..22e0e20493
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?\n",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..3d5c7b0d4a
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+headers = files('rte_bus_auxiliary.h')
+sources = files('auxiliary_common.c',
+    'auxiliary_params.c')
+if is_linux
+    sources += files('linux/auxiliary.c')
+endif
+deps += ['kvargs']
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..3146fcd5d3
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+bool auxiliary_dev_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Update a device being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param aux_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_aux_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_aux_dev
+ *	AUXILIARY device to be added before exist_aux_dev
+ * @return void
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param aux_drv
+ *      auxiliary driver
+ * @param aux_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..94f0790ba0
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_BUS_AUXILIARY_H
+#define RTE_BUS_AUXILIARY_H
+
+/**
+ * @file
+ *
+ * Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+#define RTE_BUS_AXILIARY_NAME "auxiliary"
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(rte_auxiliary_match_t) (const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(rte_auxiliary_probe_t) (struct rte_auxiliary_driver *drv,
+				    struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t *match;         /**< Device match function. */
+	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
+	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */
+	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
+	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_ ##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&(auxiliary_drv));	\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BUS_AUXILIARY_H */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..a52260657c
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,7 @@
+EXPERIMENTAL {
+	global:
+
+	# added in 21.08
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 410058de3a..45eab5233d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1

[dpdk-dev] [PATCH v4 1/2] devargs: add common key definition

[Xueming Li]

Adds common devargs key definition for "bus", "class" and "driver".

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |  2 --
 drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
 drivers/common/sfc_efx/sfc_efx.c        |  7 +++----
 drivers/common/sfc_efx/sfc_efx.h        |  2 --
 drivers/net/bonding/rte_eth_bond_args.c |  2 +-
 drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
 drivers/net/iavf/iavf_ethdev.c          |  5 ++---
 drivers/net/mlx5/mlx5.c                 |  4 ++--
 drivers/net/sfc/sfc_kvargs.c            |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
 lib/eal/common/eal_common_devargs.c     | 12 ++++++------
 lib/eal/include/rte_devargs.h           |  4 ++++
 12 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..306f2f1ab7 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -208,8 +208,6 @@ __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
 
-#define MLX5_CLASS_ARG_NAME "class"
-
 enum mlx5_class {
 	MLX5_CLASS_INVALID,
 	MLX5_CLASS_NET = RTE_BIT64(0),
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..34747c4e07 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,
 static int
 parse_class_options(const struct rte_devargs *devargs)
 {
-	const char *key = MLX5_CLASS_ARG_NAME;
+	const char *key = RTE_DEVARGS_KEY_CLASS;
 	struct rte_kvargs *kvlist;
 	int ret = 0;
 
diff --git a/drivers/common/sfc_efx/sfc_efx.c b/drivers/common/sfc_efx/sfc_efx.c
index 0b78933d9f..2dc5545760 100644
--- a/drivers/common/sfc_efx/sfc_efx.c
+++ b/drivers/common/sfc_efx/sfc_efx.c
@@ -42,7 +42,6 @@ enum sfc_efx_dev_class
 sfc_efx_dev_class_get(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvargs;
-	const char *key = SFC_EFX_KVARG_DEV_CLASS;
 	enum sfc_efx_dev_class dev_class = SFC_EFX_DEV_CLASS_NET;
 
 	if (devargs == NULL)
@@ -52,9 +51,9 @@ sfc_efx_dev_class_get(struct rte_devargs *devargs)
 	if (kvargs == NULL)
 		return dev_class;
 
-	if (rte_kvargs_count(kvargs, key) != 0) {
-		rte_kvargs_process(kvargs, key, sfc_efx_kvarg_dev_class_handler,
-				   &dev_class);
+	if (rte_kvargs_count(kvargs, RTE_DEVARGS_KEY_CLASS) != 0) {
+		rte_kvargs_process(kvargs, RTE_DEVARGS_KEY_CLASS,
+				   sfc_efx_kvarg_dev_class_handler, &dev_class);
 	}
 
 	rte_kvargs_free(kvargs);
diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
index 6b6164cb1f..c16eca60f3 100644
--- a/drivers/common/sfc_efx/sfc_efx.h
+++ b/drivers/common/sfc_efx/sfc_efx.h
@@ -19,8 +19,6 @@
 extern "C" {
 #endif
 
-#define SFC_EFX_KVARG_DEV_CLASS	"class"
-
 enum sfc_efx_dev_class {
 	SFC_EFX_DEV_CLASS_INVALID = 0,
 	SFC_EFX_DEV_CLASS_NET,
diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
index 764b1b8c8e..5406e1c934 100644
--- a/drivers/net/bonding/rte_eth_bond_args.c
+++ b/drivers/net/bonding/rte_eth_bond_args.c
@@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
 	PMD_BOND_SOCKET_ID_KVARG,
 	PMD_BOND_MAC_ADDR_KVARG,
 	PMD_BOND_AGG_MODE_KVARG,
-	"driver",
+	RTE_DEVARGS_KEY_DRIVER,
 	NULL
 };
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index cb898bdb68..1d8ca42a0f 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1660,7 +1660,6 @@ static int
 i40evf_driver_selected(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (devargs == NULL)
@@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       i40evf_check_driver_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index d688c31cfb..4e79319017 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -2440,7 +2440,6 @@ static int
 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (device_id != IAVF_DEV_ID_VF &&
@@ -2456,13 +2455,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       iavf_drv_i40evf_check_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..d0faa45944 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->max_dump_files_num = tmp;
 	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
 		config->lro.timeout = tmp;
-	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
+	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
 		DRV_LOG(DEBUG, "class argument is %s.", val);
 	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
 		config->log_hp_size = tmp;
@@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_REPRESENTOR,
 		MLX5_MAX_DUMP_FILES_NUM,
 		MLX5_LRO_TIMEOUT_USEC,
-		MLX5_CLASS_ARG_NAME,
+		RTE_DEVARGS_KEY_CLASS,
 		MLX5_HP_BUF_SIZE,
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 0efa92ed28..974c05e68e 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
 		SFC_KVARG_TX_DATAPATH,
 		SFC_KVARG_FW_VARIANT,
 		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
-		SFC_EFX_KVARG_DEV_CLASS,
+		RTE_DEVARGS_KEY_CLASS,
 		NULL,
 	};
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index e5e03e6582..8b5bfd8c3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 	unsigned long tmp;
 	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 
-	if (strcmp(key, "class") == 0)
+	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
 		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
index b31ac879a9..23aaf8b7e4 100644
--- a/lib/eal/common/eal_common_devargs.c
+++ b/lib/eal/common/eal_common_devargs.c
@@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		const char *str;
 		struct rte_kvargs *kvlist;
 	} layers[] = {
-		{ "bus=",    NULL, NULL, },
-		{ "class=",  NULL, NULL, },
-		{ "driver=", NULL, NULL, },
+		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
+		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
+		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
 	};
 	struct rte_kvargs_pair *kv = NULL;
 	struct rte_class *cls = NULL;
@@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		if (layers[i].kvlist == NULL)
 			continue;
 		kv = &layers[i].kvlist->pairs[0];
-		if (strcmp(kv->key, "bus") == 0) {
+		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
 			bus = rte_bus_find_by_name(kv->value);
 			if (bus == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
@@ -126,7 +126,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "class") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
 			cls = rte_class_find_by_name(kv->value);
 			if (cls == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
@@ -134,7 +134,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "driver") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
 			/* Ignore */
 			continue;
 		}
diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h
index 1e595b3c51..14fe7f70b6 100644
--- a/lib/eal/include/rte_devargs.h
+++ b/lib/eal/include/rte_devargs.h
@@ -25,6 +25,10 @@ extern "C" {
 #include <rte_compat.h>
 #include <rte_bus.h>
 
+#define RTE_DEVARGS_KEY_BUS "bus"
+#define RTE_DEVARGS_KEY_CLASS "class"
+#define RTE_DEVARGS_KEY_DRIVER "driver"
+
 /**
  * Type of generic device
  */
-- 
2.25.1

[dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary bus [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

Devargs syntax of auxiliary device:
  -a auxiliary:<name>[,args...]

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
Cc: Thomas Monjalon <thomas@monjalon.net>
Cc: Kinsella Ray <mdr@ashroe.eu>
---
 MAINTAINERS                               |   5 +
 doc/guides/rel_notes/release_21_08.rst    |   7 +
 drivers/bus/auxiliary/auxiliary_common.c  | 419 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 142 ++++++++
 drivers/bus/auxiliary/meson.build         |  11 +
 drivers/bus/auxiliary/private.h           | 112 ++++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
 drivers/bus/auxiliary/version.map         |   7 +
 drivers/bus/meson.build                   |   1 +
 10 files changed, 963 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..eaf691ca6a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..b335064963 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,13 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added auxiliary bus support.**
+
+  * Auxiliary bus provides a way to split function into child-devices
+    representing sub-domains of functionality. Each auxiliary device
+    represents a part of its parent functionality.
+  * Devargs syntax of auxiliary device: -a auxiliary:<name>[,args...]
+
 
 Removed Items
 -------------
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..91008bdc80
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,419 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AXILIARY_NAME, devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+/**
+ * Update a device being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary driver and device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((dr == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* The device is not blocked; Check if driver supports it. */
+	if (!auxiliary_match(dr, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	/* No initialization when marked as blocked, return without error. */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(WARNING, "  Invalid NUMA socket, default to 0\n");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "  Probe driver: %s\n", dr->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((dr->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "  Expecting VA IOVA mode but current mode is PA, not initializing\n");
+		return -EINVAL;
+	}
+
+	dev->driver = dr;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)\n",
+		      dr->driver.name, dev->name, dev->device.numa_node);
+	ret = dr->probe(dr, dev);
+	if (ret)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &dr->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	dr = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
+		      dev->name, dr->driver.name);
+
+	if (dr->remove) {
+		ret = dr->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *dr;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (!dr->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe() function for
+ *
+ * all registered drivers that have a matching entry in its id_table
+ * for discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARYBUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used\n",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *dr = NULL;
+	const char **out = addr;
+
+	/* Allow dummy name to prevent bus scan. */
+	if (strlen(name) == 0)
+		return 0;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(dr) {
+		if (dr->match(name))
+			break;
+	}
+	if (dr != NULL && addr != NULL)
+		*out = name;
+	return dr != NULL ? 0 : -1;
+}
+
+/* register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map)
+		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || !aux_dev->driver) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap)
+		return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+	rte_errno = ENOTSUP;
+	return -1;
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+		if (drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..5a1b029839
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..22e0e20493
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?\n",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARYBUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..3d5c7b0d4a
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+headers = files('rte_bus_auxiliary.h')
+sources = files('auxiliary_common.c',
+    'auxiliary_params.c')
+if is_linux
+    sources += files('linux/auxiliary.c')
+endif
+deps += ['kvargs']
+
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..3146fcd5d3
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,112 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
+		__func__, ##args)
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARYBUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARYBUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+bool auxiliary_dev_exists(const char *name);
+
+/**
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ *
+ * @return
+ *  0 on success, negative on error
+ */
+int auxiliary_scan(void);
+
+/**
+ * Update a device being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ *
+ * @param name
+ *	Auxiliary name of device to be validated
+ * @return
+ *	true: if device is to be ignored,
+ *	false: if device is to be scanned,
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/**
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ *
+ * @param aux_dev
+ *	AUXILIARY device to add
+ * @return void
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ *
+ * @param exist_aux_dev
+ *	Existing auxiliary device in auxiliary bus
+ * @param new_aux_dev
+ *	AUXILIARY device to be added before exist_aux_dev
+ * @return void
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/**
+ * Match the auxiliary Driver and Device by driver function
+ *
+ * @param aux_drv
+ *      auxiliary driver
+ * @param aux_dev
+ *      auxiliary device to match against the driver
+ * @return
+ *      the driver can handle the device
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/**
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ *
+ * @param start
+ *   Iteration starting point.
+ * @param str
+ *   Device string to match against.
+ * @param it
+ *   (unused) iterator structure.
+ * @return
+ *   A pointer to the next matching device if any.
+ *   NULL otherwise.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..94f0790ba0
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_BUS_AUXILIARY_H
+#define RTE_BUS_AUXILIARY_H
+
+/**
+ * @file
+ *
+ * Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+#define RTE_BUS_AXILIARY_NAME "auxiliary"
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(rte_auxiliary_match_t) (const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(rte_auxiliary_probe_t) (struct rte_auxiliary_driver *drv,
+				    struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_device device;                 /**< Inherit core device */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t *match;         /**< Device match function. */
+	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
+	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */
+	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
+	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_ ##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&(auxiliary_drv));	\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BUS_AUXILIARY_H */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..a52260657c
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,7 @@
+EXPERIMENTAL {
+	global:
+
+	# added in 21.08
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 410058de3a..45eab5233d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1

Re: [dpdk-dev] [PATCH v4 1/2] devargs: add common key definition

[Thomas Monjalon]

13/06/2021 14:58, Xueming Li:
> --- a/lib/eal/include/rte_devargs.h
> +++ b/lib/eal/include/rte_devargs.h
> @@ -25,6 +25,10 @@ extern "C" {
>  #include <rte_compat.h>
>  #include <rte_bus.h>
>  
> +#define RTE_DEVARGS_KEY_BUS "bus"
> +#define RTE_DEVARGS_KEY_CLASS "class"
> +#define RTE_DEVARGS_KEY_DRIVER "driver"

These macros must be documented as part of the API.
It must be said how it is used.

Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

13/06/2021 14:58, Xueming Li:
> Auxiliary bus [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> Devargs syntax of auxiliary device:
>   -a auxiliary:<name>[,args...]

What about suggesting the new generic syntax?

> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
[...]
> --- a/doc/guides/rel_notes/release_21_08.rst
> +++ b/doc/guides/rel_notes/release_21_08.rst
> @@ -55,6 +55,13 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
>  
> +* **Added auxiliary bus support.**
> +
> +  * Auxiliary bus provides a way to split function into child-devices
> +    representing sub-domains of functionality. Each auxiliary device
> +    represents a part of its parent functionality.
> +  * Devargs syntax of auxiliary device: -a auxiliary:<name>[,args...]

I am not sure the release notes are the right place to provide
a guide of the syntax, and this syntax is not the new generice one
with "bus=" that we want to promote.
I would just remove this last line from the release notes.

> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
> @@ -0,0 +1,419 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd

I think we should use the NVIDIA copyright now.

> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name)
> +{
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AXILIARY_NAME, devargs) {

Missing an "U" in RTE_BUS_AXILIARY_NAME

[...]
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list

Simpler: Scan the devices in the auxiliary bus.

[...]
> +/**
> + * Update a device being scanned.

Not clear what is updated.
It seems to be just the devargs part?

> + *
> + * @param aux_dev
> + *	AUXILIARY device.
> + */

Should not be a doxygen comment.

> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
> +{
> +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> +}

[...]
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> +			       struct rte_auxiliary_device *dev)
> +{
> +	enum rte_iova_mode iova_mode;
> +	int ret;
> +
> +	if ((dr == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	/* The device is not blocked; Check if driver supports it. */

I don't understand why the comment about "not blocked" here.
The policy check is below.

> +	if (!auxiliary_match(dr, dev))
> +		/* Match of device and driver failed */
> +		return 1;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	/* No initialization when marked as blocked, return without error. */
> +	if (dev->device.devargs != NULL &&
> +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> +		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");

Please no indent inside logs.
And no \n as it is already in the macro.

> +		return -1;
> +	}

[...]
> +static int
> +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *dr;

Not sure this variable is needed.
If you keep it, please "drv" is better.

> +	int ret = 0;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	dr = dev->driver;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +		      dev->name, dev->device.numa_node);
> +
> +	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
> +		      dev->name, dr->driver.name);
> +
> +	if (dr->remove) {
> +		ret = dr->remove(dev);
> +		if (ret < 0)
> +			return ret;
> +	}

[...]
> +/*
> + * Scan the content of the auxiliary bus, and call the probe() function for
> + *
> + * all registered drivers that have a matching entry in its id_table
> + * for discovered devices.

Please elaborate what is the id_table.

[...]
> +static int
> +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> +{
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || !aux_dev->driver) {

For all pointers, please compare with NULL, they are not booleans.

> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_map)
> +		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
> +	rte_errno = ENOTSUP;
> +	return -1;

I would prever the reverse logic: error first
and callback return at last.

[...]
Some code is not reviewed here to not make this mail too long.
[...]

> --- /dev/null
> +++ b/drivers/bus/auxiliary/meson.build
> @@ -0,0 +1,11 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2021 Mellanox Technologies, Ltd
> +
> +headers = files('rte_bus_auxiliary.h')
> +sources = files('auxiliary_common.c',
> +    'auxiliary_params.c')

I think it should with a comma and the parenthesis on next line.
Please check style of other meson files which were re-styled recently.

> +if is_linux
> +    sources += files('linux/auxiliary.c')
> +endif
> +deps += ['kvargs']
> +

Empty line at EOF

> --- /dev/null
> +++ b/drivers/bus/auxiliary/private.h
> @@ -0,0 +1,112 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2021 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef _AUXILIARY_PRIVATE_H_
> +#define _AUXILIARY_PRIVATE_H_
> +
> +#include <stdbool.h>
> +#include <stdio.h>

An empty line is missing here.

> +#include "rte_bus_auxiliary.h"
> +
> +extern struct rte_auxiliary_bus auxiliary_bus;
> +extern int auxiliary_bus_logtype;
> +
> +#define AUXILIARY_LOG(level, fmt, args...) \
> +	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
> +		__func__, ##args)

I suggest this better (pedantic-compliant) format:
#define AUXILIARY_LOG(level, ...) \
    rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, RTE_FMT("auxiliary bus: " \
        RTE_FMT_HEAD(__VA_ARGS__,) "\n", RTE_FMT_TAIL(__VA_ARGS__,)))

I think the __func__ should not be needed if log is well written.

> +
> +/* Auxiliary bus iterators */
> +#define FOREACH_DEVICE_ON_AUXILIARYBUS(p) \
> +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> +
> +#define FOREACH_DRIVER_ON_AUXILIARYBUS(p) \
> +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)

An underscore is missing between AUXILIARY and BUS.

> +
> +bool auxiliary_dev_exists(const char *name);
> +
> +/**
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + *
> + * @return
> + *  0 on success, negative on error
> + */

You can make the comments shorter as it is private (no doxygen).

> +int auxiliary_scan(void);

[...]
> + * @return void

Especially this comment is useless :)

[...]
> --- /dev/null
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
[...]
> +typedef bool(rte_auxiliary_match_t) (const char *name);

I think checkpatch will complain about the space between parens.

[...]
> +struct rte_auxiliary_device {
> +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> +	struct rte_device device;                 /**< Inherit core device */

core device should be before the name.

> +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */

Why in probing?
I suggest "Device driver"

> +};
> +
> +/** List of auxiliary devices */
> +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> +/** List of auxiliary drivers */
> +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> +
> +/**
> + * Structure describing the auxiliary bus
> + */
> +struct rte_auxiliary_bus {
> +	struct rte_bus bus;                  /**< Inherit the generic class */
> +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
> +};
> +
> +/**
> + * A structure describing an auxiliary driver.
> + */
> +struct rte_auxiliary_driver {
> +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> +	struct rte_driver driver;            /**< Inherit core driver. */
> +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> +	rte_auxiliary_match_t *match;         /**< Device match function. */
> +	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
> +	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */
> +	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
> +	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap function. */
> +	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */

Wrong search/replace missing capital letters.

[...]
> --- /dev/null
> +++ b/drivers/bus/auxiliary/version.map
> @@ -0,0 +1,7 @@
> +EXPERIMENTAL {
> +	global:
> +
> +	# added in 21.08
> +	rte_auxiliary_register;
> +	rte_auxiliary_unregister;
> +};

After more thoughts, shouldn't it be an internal symbol?
It is used only by DPDK drivers.

Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, June 22, 2021 12:11 AM
> To: Parav Pandit <parav@nvidia.com>; Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>; Kinsella Ray <mdr@ashroe.eu>; david.marchand@redhat.com;
> ferruh.yigit@intel.com
> Subject: Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus
> 
> 13/06/2021 14:58, Xueming Li:
> > Auxiliary bus [1] provides a way to split function into child-devices
> > representing sub-domains of functionality. Each auxiliary device
> > represents a part of its parent functionality.
> >
> > Auxiliary device is identified by unique device name, sysfs path:
> >   /sys/bus/auxiliary/devices/<name>
> >
> > Devargs syntax of auxiliary device:
> >   -a auxiliary:<name>[,args...]
> 
> What about suggesting the new generic syntax?

I'll list both.

> 
> > [1] kernel auxiliary bus document:
> > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> [...]
> > --- a/doc/guides/rel_notes/release_21_08.rst
> > +++ b/doc/guides/rel_notes/release_21_08.rst
> > @@ -55,6 +55,13 @@ New Features
> >       Also, make sure to start the actual text at the margin.
> >       =======================================================
> >
> > +* **Added auxiliary bus support.**
> > +
> > +  * Auxiliary bus provides a way to split function into child-devices
> > +    representing sub-domains of functionality. Each auxiliary device
> > +    represents a part of its parent functionality.
> > +  * Devargs syntax of auxiliary device: -a auxiliary:<name>[,args...]
> 
> I am not sure the release notes are the right place to provide a guide of the syntax, and this syntax is not the new generice one with
> "bus=" that we want to promote.
> I would just remove this last line from the release notes
> 
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/auxiliary_common.c
> > @@ -0,0 +1,419 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2021 Mellanox Technologies, Ltd
> 
> I think we should use the NVIDIA copyright now.

Good catch!

> 
> > +static struct rte_devargs *
> > +auxiliary_devargs_lookup(const char *name) {
> > +	struct rte_devargs *devargs;
> > +
> > +	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AXILIARY_NAME, devargs) {
> 
> Missing an "U" in RTE_BUS_AXILIARY_NAME
> 
> [...]
> > +/*
> > + * Scan the content of the auxiliary bus, and the devices in the
> > +devices
> > + * list
> 
> Simpler: Scan the devices in the auxiliary bus.
> 
> [...]
> > +/**
> > + * Update a device being scanned.
> 
> Not clear what is updated.
> It seems to be just the devargs part?
> 
> > + *
> > + * @param aux_dev
> > + *	AUXILIARY device.
> > + */
> 
> Should not be a doxygen comment.
> 
> > +void
> > +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev) {
> > +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> > +}
> 
> [...]
> > +static int
> > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *dr,
> > +			       struct rte_auxiliary_device *dev) {
> > +	enum rte_iova_mode iova_mode;
> > +	int ret;
> > +
> > +	if ((dr == NULL) || (dev == NULL))
> > +		return -EINVAL;
> > +
> > +	/* The device is not blocked; Check if driver supports it. */
> 
> I don't understand why the comment about "not blocked" here.
> The policy check is below.
> 
> > +	if (!auxiliary_match(dr, dev))
> > +		/* Match of device and driver failed */
> > +		return 1;
> > +
> > +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> > +		      dev->name, dev->device.numa_node);
> > +
> > +	/* No initialization when marked as blocked, return without error. */
> > +	if (dev->device.devargs != NULL &&
> > +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> > +		AUXILIARY_LOG(INFO, "  Device is blocked, not initializing\n");
> 
> Please no indent inside logs.
> And no \n as it is already in the macro.
> 
> > +		return -1;
> > +	}
> 
> [...]
> > +static int
> > +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev) {
> > +	struct rte_auxiliary_driver *dr;
> 
> Not sure this variable is needed.
> If you keep it, please "drv" is better.
> 
> > +	int ret = 0;
> > +
> > +	if (dev == NULL)
> > +		return -EINVAL;
> > +
> > +	dr = dev->driver;
> > +
> > +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> > +		      dev->name, dev->device.numa_node);
> > +
> > +	AUXILIARY_LOG(DEBUG, "  remove driver: %s %s\n",
> > +		      dev->name, dr->driver.name);
> > +
> > +	if (dr->remove) {
> > +		ret = dr->remove(dev);
> > +		if (ret < 0)
> > +			return ret;
> > +	}
> 
> [...]
> > +/*
> > + * Scan the content of the auxiliary bus, and call the probe()
> > +function for
> > + *
> > + * all registered drivers that have a matching entry in its id_table
> > + * for discovered devices.
> 
> Please elaborate what is the id_table.

Hmm, legacy code form pci bus, remove it.

> 
> [...]
> > +static int
> > +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova,
> > +size_t len) {
> > +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> > +
> > +	if (dev == NULL || !aux_dev->driver) {
> 
> For all pointers, please compare with NULL, they are not booleans.
> 
> > +		rte_errno = EINVAL;
> > +		return -1;
> > +	}
> > +	if (aux_dev->driver->dma_map)
> > +		return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
> > +	rte_errno = ENOTSUP;
> > +	return -1;
> 
> I would prever the reverse logic: error first and callback return at last.
> 
> [...]
> Some code is not reviewed here to not make this mail too long.
> [...]
> 
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/meson.build
> > @@ -0,0 +1,11 @@
> > +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2021 Mellanox
> > +Technologies, Ltd
> > +
> > +headers = files('rte_bus_auxiliary.h') sources =
> > +files('auxiliary_common.c',
> > +    'auxiliary_params.c')
> 
> I think it should with a comma and the parenthesis on next line.
> Please check style of other meson files which were re-styled recently.
> 
> > +if is_linux
> > +    sources += files('linux/auxiliary.c') endif deps += ['kvargs']
> > +
> 
> Empty line at EOF
> 
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/private.h
> > @@ -0,0 +1,112 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2021 Mellanox Technologies, Ltd  */
> > +
> > +#ifndef _AUXILIARY_PRIVATE_H_
> > +#define _AUXILIARY_PRIVATE_H_
> > +
> > +#include <stdbool.h>
> > +#include <stdio.h>
> 
> An empty line is missing here.
> 
> > +#include "rte_bus_auxiliary.h"
> > +
> > +extern struct rte_auxiliary_bus auxiliary_bus; extern int
> > +auxiliary_bus_logtype;
> > +
> > +#define AUXILIARY_LOG(level, fmt, args...) \
> > +	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, "%s(): " fmt "\n", \
> > +		__func__, ##args)
> 
> I suggest this better (pedantic-compliant) format:
> #define AUXILIARY_LOG(level, ...) \
>     rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, RTE_FMT("auxiliary bus: " \
>         RTE_FMT_HEAD(__VA_ARGS__,) "\n", RTE_FMT_TAIL(__VA_ARGS__,)))
> 
> I think the __func__ should not be needed if log is well written.

Thanks!

> 
> > +
> > +/* Auxiliary bus iterators */
> > +#define FOREACH_DEVICE_ON_AUXILIARYBUS(p) \
> > +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> > +
> > +#define FOREACH_DRIVER_ON_AUXILIARYBUS(p) \
> > +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> 
> An underscore is missing between AUXILIARY and BUS.
> 
> > +
> > +bool auxiliary_dev_exists(const char *name);
> > +
> > +/**
> > + * Scan the content of the auxiliary bus, and the devices in the
> > +devices
> > + * list
> > + *
> > + * @return
> > + *  0 on success, negative on error
> > + */
> 
> You can make the comments shorter as it is private (no doxygen).
> 
> > +int auxiliary_scan(void);
> 
> [...]
> > + * @return void
> 
> Especially this comment is useless :)
> 
> [...]
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> [...]
> > +typedef bool(rte_auxiliary_match_t) (const char *name);
> 
> I think checkpatch will complain about the space between parens.
> 
> [...]
> > +struct rte_auxiliary_device {
> > +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> > +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> > +	struct rte_device device;                 /**< Inherit core device */
> 
> core device should be before the name.
> 
> > +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> 
> Why in probing?
> I suggest "Device driver"

A SF device could be probed by a class driver t then another class driver, the driver field will be overridden by later probe.
Will change to "last device driver"

> 
> > +};
> > +
> > +/** List of auxiliary devices */
> > +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> > +/** List of auxiliary drivers */
> > +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> > +
> > +/**
> > + * Structure describing the auxiliary bus  */ struct
> > +rte_auxiliary_bus {
> > +	struct rte_bus bus;                  /**< Inherit the generic class */
> > +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> > +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers
> > +*/ };
> > +
> > +/**
> > + * A structure describing an auxiliary driver.
> > + */
> > +struct rte_auxiliary_driver {
> > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > +	struct rte_driver driver;            /**< Inherit core driver. */
> > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > +	rte_auxiliary_match_t *match;         /**< Device match function. */
> > +	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
> > +	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */
> > +	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
> > +	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap function. */
> > +	uint32_t drv_flags;                  /**< Flags RTE_auxiliary_DRV_*. */
> 
> Wrong search/replace missing capital letters.
> 
> [...]
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/version.map
> > @@ -0,0 +1,7 @@
> > +EXPERIMENTAL {
> > +	global:
> > +
> > +	# added in 21.08
> > +	rte_auxiliary_register;
> > +	rte_auxiliary_unregister;
> > +};
> 
> After more thoughts, shouldn't it be an internal symbol?
> It is used only by DPDK drivers.
> 

So users will not be able to compose their own driver and register with auxiliary bus?z

Agree with all other great comments, thanks!

[dpdk-dev] [PATCH v5 1/2] devargs: add common key definition

[Xueming Li]

Adds common devargs key definition for "bus", "class" and "driver".

Cc: Thomas Monjalon <thomas@monjalon.net>
Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |  2 --
 drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
 drivers/common/sfc_efx/sfc_efx.c        |  7 +++----
 drivers/common/sfc_efx/sfc_efx.h        |  2 --
 drivers/net/bonding/rte_eth_bond_args.c |  2 +-
 drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
 drivers/net/iavf/iavf_ethdev.c          |  5 ++---
 drivers/net/mlx5/mlx5.c                 |  4 ++--
 drivers/net/sfc/sfc_kvargs.c            |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
 lib/eal/common/eal_common_devargs.c     | 12 ++++++------
 lib/eal/include/rte_devargs.h           | 24 ++++++++++++++++++++++++
 12 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..306f2f1ab7 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -208,8 +208,6 @@ __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
 
-#define MLX5_CLASS_ARG_NAME "class"
-
 enum mlx5_class {
 	MLX5_CLASS_INVALID,
 	MLX5_CLASS_NET = RTE_BIT64(0),
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..34747c4e07 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,
 static int
 parse_class_options(const struct rte_devargs *devargs)
 {
-	const char *key = MLX5_CLASS_ARG_NAME;
+	const char *key = RTE_DEVARGS_KEY_CLASS;
 	struct rte_kvargs *kvlist;
 	int ret = 0;
 
diff --git a/drivers/common/sfc_efx/sfc_efx.c b/drivers/common/sfc_efx/sfc_efx.c
index 0b78933d9f..2dc5545760 100644
--- a/drivers/common/sfc_efx/sfc_efx.c
+++ b/drivers/common/sfc_efx/sfc_efx.c
@@ -42,7 +42,6 @@ enum sfc_efx_dev_class
 sfc_efx_dev_class_get(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvargs;
-	const char *key = SFC_EFX_KVARG_DEV_CLASS;
 	enum sfc_efx_dev_class dev_class = SFC_EFX_DEV_CLASS_NET;
 
 	if (devargs == NULL)
@@ -52,9 +51,9 @@ sfc_efx_dev_class_get(struct rte_devargs *devargs)
 	if (kvargs == NULL)
 		return dev_class;
 
-	if (rte_kvargs_count(kvargs, key) != 0) {
-		rte_kvargs_process(kvargs, key, sfc_efx_kvarg_dev_class_handler,
-				   &dev_class);
+	if (rte_kvargs_count(kvargs, RTE_DEVARGS_KEY_CLASS) != 0) {
+		rte_kvargs_process(kvargs, RTE_DEVARGS_KEY_CLASS,
+				   sfc_efx_kvarg_dev_class_handler, &dev_class);
 	}
 
 	rte_kvargs_free(kvargs);
diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
index 6b6164cb1f..c16eca60f3 100644
--- a/drivers/common/sfc_efx/sfc_efx.h
+++ b/drivers/common/sfc_efx/sfc_efx.h
@@ -19,8 +19,6 @@
 extern "C" {
 #endif
 
-#define SFC_EFX_KVARG_DEV_CLASS	"class"
-
 enum sfc_efx_dev_class {
 	SFC_EFX_DEV_CLASS_INVALID = 0,
 	SFC_EFX_DEV_CLASS_NET,
diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
index 764b1b8c8e..5406e1c934 100644
--- a/drivers/net/bonding/rte_eth_bond_args.c
+++ b/drivers/net/bonding/rte_eth_bond_args.c
@@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
 	PMD_BOND_SOCKET_ID_KVARG,
 	PMD_BOND_MAC_ADDR_KVARG,
 	PMD_BOND_AGG_MODE_KVARG,
-	"driver",
+	RTE_DEVARGS_KEY_DRIVER,
 	NULL
 };
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 385ebedcd3..0cfe13b7b2 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1660,7 +1660,6 @@ static int
 i40evf_driver_selected(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (devargs == NULL)
@@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       i40evf_check_driver_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index a7ef7a6d4d..6793bcef08 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -2448,7 +2448,6 @@ static int
 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (device_id != IAVF_DEV_ID_VF &&
@@ -2464,13 +2463,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       iavf_drv_i40evf_check_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..d0faa45944 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->max_dump_files_num = tmp;
 	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
 		config->lro.timeout = tmp;
-	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
+	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
 		DRV_LOG(DEBUG, "class argument is %s.", val);
 	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
 		config->log_hp_size = tmp;
@@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_REPRESENTOR,
 		MLX5_MAX_DUMP_FILES_NUM,
 		MLX5_LRO_TIMEOUT_USEC,
-		MLX5_CLASS_ARG_NAME,
+		RTE_DEVARGS_KEY_CLASS,
 		MLX5_HP_BUF_SIZE,
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 0efa92ed28..974c05e68e 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
 		SFC_KVARG_TX_DATAPATH,
 		SFC_KVARG_FW_VARIANT,
 		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
-		SFC_EFX_KVARG_DEV_CLASS,
+		RTE_DEVARGS_KEY_CLASS,
 		NULL,
 	};
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index e5e03e6582..8b5bfd8c3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 	unsigned long tmp;
 	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 
-	if (strcmp(key, "class") == 0)
+	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
 		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
index b31ac879a9..23aaf8b7e4 100644
--- a/lib/eal/common/eal_common_devargs.c
+++ b/lib/eal/common/eal_common_devargs.c
@@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		const char *str;
 		struct rte_kvargs *kvlist;
 	} layers[] = {
-		{ "bus=",    NULL, NULL, },
-		{ "class=",  NULL, NULL, },
-		{ "driver=", NULL, NULL, },
+		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
+		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
+		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
 	};
 	struct rte_kvargs_pair *kv = NULL;
 	struct rte_class *cls = NULL;
@@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		if (layers[i].kvlist == NULL)
 			continue;
 		kv = &layers[i].kvlist->pairs[0];
-		if (strcmp(kv->key, "bus") == 0) {
+		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
 			bus = rte_bus_find_by_name(kv->value);
 			if (bus == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
@@ -126,7 +126,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "class") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
 			cls = rte_class_find_by_name(kv->value);
 			if (cls == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
@@ -134,7 +134,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "driver") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
 			/* Ignore */
 			continue;
 		}
diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h
index 1e595b3c51..cd90944fe8 100644
--- a/lib/eal/include/rte_devargs.h
+++ b/lib/eal/include/rte_devargs.h
@@ -25,6 +25,30 @@ extern "C" {
 #include <rte_compat.h>
 #include <rte_bus.h>
 
+/**
+ * Bus type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't use this key as bus type
+ * is resolved as first optional value separated by ":".
+ */
+#define RTE_DEVARGS_KEY_BUS "bus"
+
+/**
+ * Class type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't parse class type. PMD driver is
+ * encouraged to use this key to resolve class type.
+ */
+#define RTE_DEVARGS_KEY_CLASS "class"
+
+/**
+ * Driver type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't parse driver type. PMD driver is
+ * encouraged to use this key to resolve driver type.
+ */
+#define RTE_DEVARGS_KEY_DRIVER "driver"
+
 /**
  * Type of generic device
  */
-- 
2.25.1

[dpdk-dev] [PATCH v5 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary bus [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

Devargs legacy syntax ofauxiliary device:
  -a auxiliary:<name>[,args...]
Devargs generic syntax of auxiliary device:
  -a bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
Cc: Thomas Monjalon <thomas@monjalon.net>
Cc: Kinsella Ray <mdr@ashroe.eu>
---
 MAINTAINERS                               |   5 +
 doc/guides/rel_notes/release_21_08.rst    |   6 +
 drivers/bus/auxiliary/auxiliary_common.c  | 418 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  58 +++
 drivers/bus/auxiliary/linux/auxiliary.c   | 142 ++++++++
 drivers/bus/auxiliary/meson.build         |  16 +
 drivers/bus/auxiliary/private.h           |  75 ++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 199 ++++++++++
 drivers/bus/auxiliary/version.map         |   3 +
 drivers/bus/meson.build                   |   1 +
 10 files changed, 923 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..eaf691ca6a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..e7ef4c8a05 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added auxiliary bus support.**
+
+  Auxiliary bus provides a way to split function into child-devices
+  representing sub-domains of functionality. Each auxiliary device
+  represents a part of its parent functionality.
+
 
 Removed Items
 -------------
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..25d6802f24
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,418 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the devices in the auxiliary bus.
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+/*
+ * Update a device's devargs being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary driver and device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((drv == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* Check if driver supports it. */
+	if (!auxiliary_match(drv, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i",
+		      dev->name, dev->device.numa_node);
+
+	/* No initialization when marked as blocked, return without error. */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting socket to 0");
+		dev->device.numa_node = 0;
+	}
+
+	AUXILIARY_LOG(DEBUG, "Probe driver: %s", drv->driver.name);
+
+	iova_mode = rte_eal_iova_mode();
+	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "Expecting VA IOVA mode but current mode is PA, not initializing");
+		return -EINVAL;
+	}
+
+	dev->driver = drv;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)",
+		      drv->driver.name, dev->name, dev->device.numa_node);
+	ret = drv->probe(drv, dev);
+	if (ret != 0)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &drv->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *drv;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	drv = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i",
+		      dev->name, dev->device.numa_node);
+
+	AUXILIARY_LOG(DEBUG, "remove driver: %s %s",
+		      dev->name, drv->driver.name);
+
+	if (drv->remove != NULL) {
+		ret = drv->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *drv;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if (!drv->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(drv, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe function for
+ * all registered drivers to try to probe discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *drv = NULL;
+	const char **out = addr;
+
+	/* Allow dummy name to prevent bus scan. */
+	if (strlen(name) == 0)
+		return 0;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if (drv->match(name))
+			break;
+	}
+	if (drv != NULL && addr != NULL)
+		*out = name;
+	return drv != NULL ? 0 : -1;
+}
+
+/* Register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* Unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || aux_dev->driver == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map == NULL) {
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || aux_dev->driver == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap == NULL) {
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+}
+
+bool
+auxiliary_ignore_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..dc63b2e574
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..1cc075cd13
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get numa node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_ignore_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..357550eff7
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+headers = files(
+        'rte_bus_auxiliary.h',
+)
+sources = files(
+        'auxiliary_common.c',
+        'auxiliary_params.c',
+)
+if is_linux
+    sources += files(
+        'linux/auxiliary.c',
+    )
+endif
+deps += ['kvargs']
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..36a66dcf1b
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef _AUXILIARY_PRIVATE_H_
+#define _AUXILIARY_PRIVATE_H_
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, ...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, \
+		RTE_FMT("auxiliary bus: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
+			RTE_FMT_TAIL(__VA_ARGS__,)))
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARY_BUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARY_BUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+bool auxiliary_dev_exists(const char *name);
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list.
+ */
+int auxiliary_scan(void);
+
+/*
+ * Update a device being scanned.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ */
+bool auxiliary_ignore_device(const char *name);
+
+/*
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary Device
+ * list). This function also updates the bus references of the auxiliary
+ * Device (and the generic device object embedded within.
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/*
+ * Match the auxiliary Driver and Device by driver function
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Iterate over internal devices, matching any device against the provided
+ * string.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* _AUXILIARY_PRIVATE_H_ */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..046350bb52
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,199 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_BUS_AUXILIARY_H
+#define RTE_BUS_AUXILIARY_H
+
+/**
+ * @file
+ *
+ * Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+#define RTE_BUS_AUXILIARY_NAME "auxiliary"
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(rte_auxiliary_match_t)(const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(rte_auxiliary_probe_t)(struct rte_auxiliary_driver *drv,
+				    struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	struct rte_device device;                 /**< Inherit core device */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< Last device driver */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;            /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t *match;         /**< Device match function. */
+	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
+	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */
+	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
+	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap function. */
+	uint32_t drv_flags;                  /**< Flags RTE_AUXILIARY_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
+	RTE_INIT(auxiliaryinitfn_ ##nm)				\
+	{							\
+		(auxiliary_drv).driver.name = RTE_STR(nm);	\
+		rte_auxiliary_register(&(auxiliary_drv));	\
+	}							\
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BUS_AUXILIARY_H */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..3081f19626
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,3 @@
+EXPERIMENTAL {
+	# added in 21.08
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 410058de3a..45eab5233d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1

Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

23/06/2021 01:50, Xueming(Steven) Li:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 13/06/2021 14:58, Xueming Li:
> > > --- /dev/null
> > > +++ b/drivers/bus/auxiliary/version.map
> > > @@ -0,0 +1,7 @@
> > > +EXPERIMENTAL {
> > > +	global:
> > > +
> > > +	# added in 21.08
> > > +	rte_auxiliary_register;
> > > +	rte_auxiliary_unregister;
> > > +};
> > 
> > After more thoughts, shouldn't it be an internal symbol?
> > It is used only by DPDK drivers.
> 
> So users will not be able to compose their own driver and register with auxiliary bus?z

Yes, that's an interesting question actually.
We can continue with experimental/stable status of driver ABI,
but we should invent a new ABI flag like DRIVER,
so there is no stability policy on such symbol.

Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

23/06/2021 01:50, Xueming(Steven) Li:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 13/06/2021 14:58, Xueming Li:
> > > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> > 
> > Why in probing?
> > I suggest "Device driver"
> 
> A SF device could be probed by a class driver t then another class driver, the driver field will be overridden by later probe.
> Will change to "last device driver"

"Last" is a bit confusing.
Any field can be potentially overwritten, i.e. change its value,
and we don't comment "last value" ;)

Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, June 23, 2021 4:22 PM
> To: Parav Pandit <parav@nvidia.com>; Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>; Kinsella Ray <mdr@ashroe.eu>; david.marchand@redhat.com;
> ferruh.yigit@intel.com
> Subject: Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus
> 
> 23/06/2021 01:50, Xueming(Steven) Li:
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 13/06/2021 14:58, Xueming Li:
> > > > +	struct rte_auxiliary_driver *driver;      /**< driver used in probing */
> > >
> > > Why in probing?
> > > I suggest "Device driver"
> >
> > A SF device could be probed by a class driver t then another class driver, the driver field will be overridden by later probe.
> > Will change to "last device driver"
> 
> "Last" is a bit confusing.
> Any field can be potentially overwritten, i.e. change its value, and we don't comment "last value" ;)

OK, will reflect in v6.

Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, June 23, 2021 4:15 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: Parav Pandit <parav@nvidia.com>; dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>; Kinsella Ray <mdr@ashroe.eu>;
> david.marchand@redhat.com; ferruh.yigit@intel.com
> Subject: Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus
> 
> 23/06/2021 01:50, Xueming(Steven) Li:
> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 13/06/2021 14:58, Xueming Li:
> > > > --- /dev/null
> > > > +++ b/drivers/bus/auxiliary/version.map
> > > > @@ -0,0 +1,7 @@
> > > > +EXPERIMENTAL {
> > > > +	global:
> > > > +
> > > > +	# added in 21.08
> > > > +	rte_auxiliary_register;
> > > > +	rte_auxiliary_unregister;
> > > > +};
> > >
> > > After more thoughts, shouldn't it be an internal symbol?
> > > It is used only by DPDK drivers.
> >
> > So users will not be able to compose their own driver and register
> > with auxiliary bus?z
> 
> Yes, that's an interesting question actually.
> We can continue with experimental/stable status of driver ABI, but we should invent a new ABI flag like DRIVER, so there is no stability
> policy on such symbol.

Not quite understand here, why we want to export the function but no ABI guarantee? the api shouldn't change frequently IMHO.

Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

23/06/2021 16:52, Xueming(Steven) Li:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 23/06/2021 01:50, Xueming(Steven) Li:
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > 13/06/2021 14:58, Xueming Li:
> > > > > --- /dev/null
> > > > > +++ b/drivers/bus/auxiliary/version.map
> > > > > @@ -0,0 +1,7 @@
> > > > > +EXPERIMENTAL {
> > > > > +	global:
> > > > > +
> > > > > +	# added in 21.08
> > > > > +	rte_auxiliary_register;
> > > > > +	rte_auxiliary_unregister;
> > > > > +};
> > > >
> > > > After more thoughts, shouldn't it be an internal symbol?
> > > > It is used only by DPDK drivers.
> > >
> > > So users will not be able to compose their own driver and register
> > > with auxiliary bus?z
> > 
> > Yes, that's an interesting question actually.
> > We can continue with experimental/stable status of driver ABI, but we should invent a new ABI flag like DRIVER, so there is no stability
> > policy on such symbol.
> 
> Not quite understand here, why we want to export the function but no ABI guarantee? the api shouldn't change frequently IMHO.

Sorry my message was not clear.
I am OK to keep "EXPERIMENTAL" in this patch.
But in future, we don't want to make driver interface as part
of the stable ABI because it makes evolution harder for no good reason:
nobody is asking for a stable interface with drivers.

Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

Thanks for clarification, will update in next version.
________________________________
From: Thomas Monjalon <thomas@monjalon.net>
Sent: Thursday, June 24, 2021 2:37:19 PM
To: Xueming(Steven) Li <xuemingl@nvidia.com>
Cc: Parav Pandit <parav@nvidia.com>; dev@dpdk.org <dev@dpdk.org>; Wang Haiyue <haiyue.wang@intel.com>; Kinsella Ray <mdr@ashroe.eu>; david.marchand@redhat.com <david.marchand@redhat.com>; ferruh.yigit@intel.com <ferruh.yigit@intel.com>
Subject: Re: [dpdk-dev] [PATCH v4 2/2] bus/auxiliary: introduce auxiliary bus

23/06/2021 16:52, Xueming(Steven) Li:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 23/06/2021 01:50, Xueming(Steven) Li:
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > 13/06/2021 14:58, Xueming Li:
> > > > > --- /dev/null
> > > > > +++ b/drivers/bus/auxiliary/version.map
> > > > > @@ -0,0 +1,7 @@
> > > > > +EXPERIMENTAL {
> > > > > +     global:
> > > > > +
> > > > > +     # added in 21.08
> > > > > +     rte_auxiliary_register;
> > > > > +     rte_auxiliary_unregister;
> > > > > +};
> > > >
> > > > After more thoughts, shouldn't it be an internal symbol?
> > > > It is used only by DPDK drivers.
> > >
> > > So users will not be able to compose their own driver and register
> > > with auxiliary bus?z
> >
> > Yes, that's an interesting question actually.
> > We can continue with experimental/stable status of driver ABI, but we should invent a new ABI flag like DRIVER, so there is no stability
> > policy on such symbol.
>
> Not quite understand here, why we want to export the function but no ABI guarantee? the api shouldn't change frequently IMHO.

Sorry my message was not clear.
I am OK to keep "EXPERIMENTAL" in this patch.
But in future, we don't want to make driver interface as part
of the stable ABI because it makes evolution harder for no good reason:
nobody is asking for a stable interface with drivers.

Re: [dpdk-dev] [PATCH v5 1/2] devargs: add common key definition

[Thomas Monjalon]

23/06/2021 02:03, Xueming Li:
> Adds common devargs key definition for "bus", "class" and "driver".
> 
> Cc: Thomas Monjalon <thomas@monjalon.net>
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>

Acked-by: Thomas Monjalon <thomas@monjalon.net>

Re: [dpdk-dev] [PATCH v5 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

23/06/2021 02:03, Xueming Li:
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
> +			       struct rte_auxiliary_device *dev)
[...]
> +	AUXILIARY_LOG(DEBUG, "Probe driver: %s", drv->driver.name);

I think the debug log above is useless given the ones below.

> +
> +	iova_mode = rte_eal_iova_mode();
> +	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> +	    iova_mode != RTE_IOVA_VA) {
> +		AUXILIARY_LOG(ERR, "Expecting VA IOVA mode but current mode is PA, not initializing");

The error log should mention which driver is requesting VA and not initializing.

> +		return -EINVAL;
> +	}
> +
> +	dev->driver = drv;
> +
> +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)",
> +		      drv->driver.name, dev->name, dev->device.numa_node);
> +	ret = drv->probe(drv, dev);
> +	if (ret != 0)
> +		dev->driver = NULL;
> +	else
> +		dev->device.driver = &drv->driver;
> +
> +	return ret;
> +}
> +
> +/*
> + * Call the remove() function of the driver.
> + */
> +static int
> +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *drv;
> +	int ret = 0;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	drv = dev->driver;
> +
> +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i",
> +		      dev->name, dev->device.numa_node);
> +
> +	AUXILIARY_LOG(DEBUG, "remove driver: %s %s",
> +		      dev->name, drv->driver.name);

Better to merge both logs together.

> +
> +	if (drv->remove != NULL) {
> +		ret = drv->remove(dev);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	/* clear driver structure */
> +	dev->driver = NULL;
> +	dev->device.driver = NULL;
> +
> +	return 0;
> +}

[...]
> +static int
> +auxiliary_parse(const char *name, void *addr)
> +{
> +	struct rte_auxiliary_driver *drv = NULL;
> +	const char **out = addr;
> +
> +	/* Allow dummy name to prevent bus scan. */
> +	if (strlen(name) == 0)
> +		return 0;

Which syntax is it?

[...]
> +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> +		if (kvargs == NULL) {
> +			RTE_LOG(ERR, EAL, "cannot parse argument list\n");

Should use AUXILIARY_LOG.

[...]
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> +	/* Get numa node, default to 0 if not present */

s/numa/NUMA/

[...]
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list
> + */

Simpler: Scan the devices in the auxiliary bus.

> +int
> +auxiliary_scan(void)
> +{

[...]
> +++ b/drivers/bus/auxiliary/private.h
> @@ -0,0 +1,75 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> + */
> +
> +#ifndef _AUXILIARY_PRIVATE_H_
> +#define _AUXILIARY_PRIVATE_H_

nit: we can avoid the first and final underscores

[...]
> +/*
> + * Validate whether a device with given auxiliary device should be ignored
> + * or not.
> + */
> +bool auxiliary_ignore_device(const char *name);

Suggestion: auxiliary_is_ignored_device() ?

> +/*
> + * Add an auxiliary device to the auxiliary bus (append to auxiliary Device

s/Device/device/

> + * list). This function also updates the bus references of the auxiliary
> + * Device (and the generic device object embedded within.

s/(and/and/

> + */
> +void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
> +
> +/*
> + * Insert an auxiliary device in the auxiliary bus at a particular location
> + * in the device list. It also updates the auxiliary bus reference of the
> + * new devices to be inserted.
> + */
> +void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			     struct rte_auxiliary_device *new_aux_dev);
> +
> +/*
> + * Match the auxiliary Driver and Device by driver function

no need of capital letters in driver and device.

> + */
> +bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		     const struct rte_auxiliary_device *aux_dev);
> +
> +/*
> + * Iterate over internal devices, matching any device against the provided

What do you mean by "internal"?

> + * string.
> + */
> +void *auxiliary_dev_iterate(const void *start, const char *str,
> +			    const struct rte_dev_iterator *it);
> +

[...]
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> +struct rte_auxiliary_driver {
> +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> +	struct rte_driver driver;            /**< Inherit core driver. */
> +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> +	rte_auxiliary_match_t *match;         /**< Device match function. */
> +	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
> +	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */

No capital in Probe and Remove

> +	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
> +	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap function. */

s/dma/DMA/
There may be other occurences to check.

> +	uint32_t drv_flags;                  /**< Flags RTE_AUXILIARY_DRV_*. */
> +};

The alignment of some comments above can be improved by one space.

[...]
> +/** Helper for auxiliary device registration from driver instance */
> +#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
> +	RTE_INIT(auxiliaryinitfn_ ##nm)				\
> +	{							\
> +		(auxiliary_drv).driver.name = RTE_STR(nm);	\
> +		rte_auxiliary_register(&(auxiliary_drv));	\
> +	}							\
> +	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)

I think it is better not trying to align backslashes in such macro,
but leave only a single space.

[...]
> +++ b/drivers/bus/auxiliary/version.map
> @@ -0,0 +1,3 @@
> +EXPERIMENTAL {
> +	# added in 21.08
> +};

Looks like you need to bring the symbols back :)

Looks good overrall, thanks.

Re: [dpdk-dev] [PATCH v5 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Friday, June 25, 2021 12:19 AM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>; Kinsella Ray <mdr@ashroe.eu>
> Subject: Re: [dpdk-dev] [PATCH v5 2/2] bus/auxiliary: introduce auxiliary bus
> 
> 23/06/2021 02:03, Xueming Li:
> > +static int
> > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
> > +			       struct rte_auxiliary_device *dev)
> [...]
> > +	AUXILIARY_LOG(DEBUG, "Probe driver: %s", drv->driver.name);
> 
> I think the debug log above is useless given the ones below.
> 
> > +
> > +	iova_mode = rte_eal_iova_mode();
> > +	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> > +	    iova_mode != RTE_IOVA_VA) {
> > +		AUXILIARY_LOG(ERR, "Expecting VA IOVA mode but current mode is PA,
> > +not initializing");
> 
> The error log should mention which driver is requesting VA and not initializing.
> 
> > +		return -EINVAL;
> > +	}
> > +
> > +	dev->driver = drv;
> > +
> > +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)",
> > +		      drv->driver.name, dev->name, dev->device.numa_node);
> > +	ret = drv->probe(drv, dev);
> > +	if (ret != 0)
> > +		dev->driver = NULL;
> > +	else
> > +		dev->device.driver = &drv->driver;
> > +
> > +	return ret;
> > +}
> > +
> > +/*
> > + * Call the remove() function of the driver.
> > + */
> > +static int
> > +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev) {
> > +	struct rte_auxiliary_driver *drv;
> > +	int ret = 0;
> > +
> > +	if (dev == NULL)
> > +		return -EINVAL;
> > +
> > +	drv = dev->driver;
> > +
> > +	AUXILIARY_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i",
> > +		      dev->name, dev->device.numa_node);
> > +
> > +	AUXILIARY_LOG(DEBUG, "remove driver: %s %s",
> > +		      dev->name, drv->driver.name);
> 
> Better to merge both logs together.
> 
> > +
> > +	if (drv->remove != NULL) {
> > +		ret = drv->remove(dev);
> > +		if (ret < 0)
> > +			return ret;
> > +	}
> > +
> > +	/* clear driver structure */
> > +	dev->driver = NULL;
> > +	dev->device.driver = NULL;
> > +
> > +	return 0;
> > +}
> 
> [...]
> > +static int
> > +auxiliary_parse(const char *name, void *addr) {
> > +	struct rte_auxiliary_driver *drv = NULL;
> > +	const char **out = addr;
> > +
> > +	/* Allow dummy name to prevent bus scan. */
> > +	if (strlen(name) == 0)
> > +		return 0;
> 
> Which syntax is it?

Allow empty device name "auxiliary:" to bypass entire auxiliary bus scan.

> 
> [...]
> > +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> > +		if (kvargs == NULL) {
> > +			RTE_LOG(ERR, EAL, "cannot parse argument list\n");
> 
> Should use AUXILIARY_LOG.
> 
> [...]
> > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> > +	/* Get numa node, default to 0 if not present */
> 
> s/numa/NUMA/
> 
> [...]
> > +/*
> > + * Scan the content of the auxiliary bus, and the devices in the
> > +devices
> > + * list
> > + */
> 
> Simpler: Scan the devices in the auxiliary bus.
> 
> > +int
> > +auxiliary_scan(void)
> > +{
> 
> [...]
> > +++ b/drivers/bus/auxiliary/private.h
> > @@ -0,0 +1,75 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright (c) 2021 NVIDIA Corporation & Affiliates  */
> > +
> > +#ifndef _AUXILIARY_PRIVATE_H_
> > +#define _AUXILIARY_PRIVATE_H_
> 
> nit: we can avoid the first and final underscores
> 
> [...]
> > +/*
> > + * Validate whether a device with given auxiliary device should be
> > +ignored
> > + * or not.
> > + */
> > +bool auxiliary_ignore_device(const char *name);
> 
> Suggestion: auxiliary_is_ignored_device() ?
> 
> > +/*
> > + * Add an auxiliary device to the auxiliary bus (append to auxiliary
> > +Device
> 
> s/Device/device/
> 
> > + * list). This function also updates the bus references of the
> > + auxiliary
> > + * Device (and the generic device object embedded within.
> 
> s/(and/and/
> 
> > + */
> > +void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
> > +
> > +/*
> > + * Insert an auxiliary device in the auxiliary bus at a particular
> > +location
> > + * in the device list. It also updates the auxiliary bus reference of
> > +the
> > + * new devices to be inserted.
> > + */
> > +void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> > +			     struct rte_auxiliary_device *new_aux_dev);
> > +
> > +/*
> > + * Match the auxiliary Driver and Device by driver function
> 
> no need of capital letters in driver and device.
> 
> > + */
> > +bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> > +		     const struct rte_auxiliary_device *aux_dev);
> > +
> > +/*
> > + * Iterate over internal devices, matching any device against the
> > +provided
> 
> What do you mean by "internal"?
> 
> > + * string.
> > + */
> > +void *auxiliary_dev_iterate(const void *start, const char *str,
> > +			    const struct rte_dev_iterator *it);
> > +
> 
> [...]
> > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > +struct rte_auxiliary_driver {
> > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > +	struct rte_driver driver;            /**< Inherit core driver. */
> > +	struct rte_auxiliary_bus *bus;       /**< Auxiliary bus reference. */
> > +	rte_auxiliary_match_t *match;         /**< Device match function. */
> > +	rte_auxiliary_probe_t *probe;         /**< Device Probe function. */
> > +	rte_auxiliary_remove_t *remove;       /**< Device Remove function. */
> 
> No capital in Probe and Remove
> 
> > +	rte_auxiliary_dma_map_t *dma_map;     /**< Device dma map function. */
> > +	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device dma unmap
> > +function. */
> 
> s/dma/DMA/
> There may be other occurences to check.
> 
> > +	uint32_t drv_flags;                  /**< Flags RTE_AUXILIARY_DRV_*. */
> > +};
> 
> The alignment of some comments above can be improved by one space.
> 
> [...]
> > +/** Helper for auxiliary device registration from driver instance */
> > +#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv)		\
> > +	RTE_INIT(auxiliaryinitfn_ ##nm)				\
> > +	{							\
> > +		(auxiliary_drv).driver.name = RTE_STR(nm);	\
> > +		rte_auxiliary_register(&(auxiliary_drv));	\
> > +	}							\
> > +	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
> 
> I think it is better not trying to align backslashes in such macro, but leave only a single space.
> 
> [...]
> > +++ b/drivers/bus/auxiliary/version.map
> > @@ -0,0 +1,3 @@
> > +EXPERIMENTAL {
> > +	# added in 21.08
> > +};
> 
> Looks like you need to bring the symbols back :)
> 
> Looks good overrall, thanks.
> 

Re: [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus

[Stephen Hemminger]

On Thu, 11 Mar 2021 21:01:13 +0800
Xueming Li <xuemingl@nvidia.com> wrote:

> +	AUXILIAR_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> +			dev->name, dev->device.numa_node);

Your log messages will be double spaced.
The macro is already  adding a new line.

> +
> +#define AUXILIAR_LOG(level, fmt, args...) \
> +	rte_log(RTE_LOG_ ## level, auxiliary_logtype_bus, "%s(): " fmt "\n", \
> +		__func__, ##args)

Re: [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Stephen Hemminger <stephen@networkplumber.org>
> Sent: Friday, June 25, 2021 12:34 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Parav Pandit <parav@nvidia.com>; Ray Kinsella
> <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: Re: [dpdk-dev] [RFC] bus/auxiliary: introduce auxiliary bus
> 
> On Thu, 11 Mar 2021 21:01:13 +0800
> Xueming Li <xuemingl@nvidia.com> wrote:
> 
> > +	AUXILIAR_LOG(DEBUG, "Auxiliary device %s on NUMA socket %i\n",
> > +			dev->name, dev->device.numa_node);
> 
> Your log messages will be double spaced.
> The macro is already  adding a new line.

Thanks, update in next version.

> 
> > +
> > +#define AUXILIAR_LOG(level, fmt, args...) \
> > +	rte_log(RTE_LOG_ ## level, auxiliary_logtype_bus, "%s(): " fmt "\n", \
> > +		__func__, ##args)

[dpdk-dev] [PATCH v6 1/2] devargs: add common key definition

[Xueming Li]

Adds common devargs key definition for "bus", "class" and "driver".

Acked-by: Thomas Monjalon <thomas@monjalon.net>
Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |  2 --
 drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
 drivers/common/sfc_efx/sfc_efx.c        |  7 +++----
 drivers/common/sfc_efx/sfc_efx.h        |  2 --
 drivers/net/bonding/rte_eth_bond_args.c |  2 +-
 drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
 drivers/net/iavf/iavf_ethdev.c          |  5 ++---
 drivers/net/mlx5/mlx5.c                 |  4 ++--
 drivers/net/sfc/sfc_kvargs.c            |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
 lib/eal/common/eal_common_devargs.c     | 12 ++++++------
 lib/eal/include/rte_devargs.h           | 24 ++++++++++++++++++++++++
 12 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..306f2f1ab7 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -208,8 +208,6 @@ __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
 
-#define MLX5_CLASS_ARG_NAME "class"
-
 enum mlx5_class {
 	MLX5_CLASS_INVALID,
 	MLX5_CLASS_NET = RTE_BIT64(0),
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..34747c4e07 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,
 static int
 parse_class_options(const struct rte_devargs *devargs)
 {
-	const char *key = MLX5_CLASS_ARG_NAME;
+	const char *key = RTE_DEVARGS_KEY_CLASS;
 	struct rte_kvargs *kvlist;
 	int ret = 0;
 
diff --git a/drivers/common/sfc_efx/sfc_efx.c b/drivers/common/sfc_efx/sfc_efx.c
index 0b78933d9f..2dc5545760 100644
--- a/drivers/common/sfc_efx/sfc_efx.c
+++ b/drivers/common/sfc_efx/sfc_efx.c
@@ -42,7 +42,6 @@ enum sfc_efx_dev_class
 sfc_efx_dev_class_get(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvargs;
-	const char *key = SFC_EFX_KVARG_DEV_CLASS;
 	enum sfc_efx_dev_class dev_class = SFC_EFX_DEV_CLASS_NET;
 
 	if (devargs == NULL)
@@ -52,9 +51,9 @@ sfc_efx_dev_class_get(struct rte_devargs *devargs)
 	if (kvargs == NULL)
 		return dev_class;
 
-	if (rte_kvargs_count(kvargs, key) != 0) {
-		rte_kvargs_process(kvargs, key, sfc_efx_kvarg_dev_class_handler,
-				   &dev_class);
+	if (rte_kvargs_count(kvargs, RTE_DEVARGS_KEY_CLASS) != 0) {
+		rte_kvargs_process(kvargs, RTE_DEVARGS_KEY_CLASS,
+				   sfc_efx_kvarg_dev_class_handler, &dev_class);
 	}
 
 	rte_kvargs_free(kvargs);
diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
index 6b6164cb1f..c16eca60f3 100644
--- a/drivers/common/sfc_efx/sfc_efx.h
+++ b/drivers/common/sfc_efx/sfc_efx.h
@@ -19,8 +19,6 @@
 extern "C" {
 #endif
 
-#define SFC_EFX_KVARG_DEV_CLASS	"class"
-
 enum sfc_efx_dev_class {
 	SFC_EFX_DEV_CLASS_INVALID = 0,
 	SFC_EFX_DEV_CLASS_NET,
diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
index 764b1b8c8e..5406e1c934 100644
--- a/drivers/net/bonding/rte_eth_bond_args.c
+++ b/drivers/net/bonding/rte_eth_bond_args.c
@@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
 	PMD_BOND_SOCKET_ID_KVARG,
 	PMD_BOND_MAC_ADDR_KVARG,
 	PMD_BOND_AGG_MODE_KVARG,
-	"driver",
+	RTE_DEVARGS_KEY_DRIVER,
 	NULL
 };
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 385ebedcd3..0cfe13b7b2 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1660,7 +1660,6 @@ static int
 i40evf_driver_selected(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (devargs == NULL)
@@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       i40evf_check_driver_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 5290588b17..472538181e 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -2448,7 +2448,6 @@ static int
 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (device_id != IAVF_DEV_ID_VF &&
@@ -2464,13 +2463,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       iavf_drv_i40evf_check_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..d0faa45944 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->max_dump_files_num = tmp;
 	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
 		config->lro.timeout = tmp;
-	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
+	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
 		DRV_LOG(DEBUG, "class argument is %s.", val);
 	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
 		config->log_hp_size = tmp;
@@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_REPRESENTOR,
 		MLX5_MAX_DUMP_FILES_NUM,
 		MLX5_LRO_TIMEOUT_USEC,
-		MLX5_CLASS_ARG_NAME,
+		RTE_DEVARGS_KEY_CLASS,
 		MLX5_HP_BUF_SIZE,
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 0efa92ed28..974c05e68e 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
 		SFC_KVARG_TX_DATAPATH,
 		SFC_KVARG_FW_VARIANT,
 		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
-		SFC_EFX_KVARG_DEV_CLASS,
+		RTE_DEVARGS_KEY_CLASS,
 		NULL,
 	};
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index e5e03e6582..8b5bfd8c3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 	unsigned long tmp;
 	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 
-	if (strcmp(key, "class") == 0)
+	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
 		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
index b31ac879a9..23aaf8b7e4 100644
--- a/lib/eal/common/eal_common_devargs.c
+++ b/lib/eal/common/eal_common_devargs.c
@@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		const char *str;
 		struct rte_kvargs *kvlist;
 	} layers[] = {
-		{ "bus=",    NULL, NULL, },
-		{ "class=",  NULL, NULL, },
-		{ "driver=", NULL, NULL, },
+		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
+		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
+		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
 	};
 	struct rte_kvargs_pair *kv = NULL;
 	struct rte_class *cls = NULL;
@@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		if (layers[i].kvlist == NULL)
 			continue;
 		kv = &layers[i].kvlist->pairs[0];
-		if (strcmp(kv->key, "bus") == 0) {
+		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
 			bus = rte_bus_find_by_name(kv->value);
 			if (bus == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
@@ -126,7 +126,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "class") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
 			cls = rte_class_find_by_name(kv->value);
 			if (cls == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
@@ -134,7 +134,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "driver") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
 			/* Ignore */
 			continue;
 		}
diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h
index 1e595b3c51..cd90944fe8 100644
--- a/lib/eal/include/rte_devargs.h
+++ b/lib/eal/include/rte_devargs.h
@@ -25,6 +25,30 @@ extern "C" {
 #include <rte_compat.h>
 #include <rte_bus.h>
 
+/**
+ * Bus type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't use this key as bus type
+ * is resolved as first optional value separated by ":".
+ */
+#define RTE_DEVARGS_KEY_BUS "bus"
+
+/**
+ * Class type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't parse class type. PMD driver is
+ * encouraged to use this key to resolve class type.
+ */
+#define RTE_DEVARGS_KEY_CLASS "class"
+
+/**
+ * Driver type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't parse driver type. PMD driver is
+ * encouraged to use this key to resolve driver type.
+ */
+#define RTE_DEVARGS_KEY_DRIVER "driver"
+
 /**
  * Type of generic device
  */
-- 
2.25.1

[dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary bus [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

Devargs legacy syntax ofauxiliary device:
  -a auxiliary:<name>[,args...]
Devargs generic syntax of auxiliary device:
  -a bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
Cc: Thomas Monjalon <thomas@monjalon.net>
Cc: Kinsella Ray <mdr@ashroe.eu>
---
 MAINTAINERS                               |   5 +
 doc/guides/rel_notes/release_21_08.rst    |   6 +
 drivers/bus/auxiliary/auxiliary_common.c  | 411 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
 drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
 drivers/bus/auxiliary/meson.build         |  16 +
 drivers/bus/auxiliary/private.h           |  74 ++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
 drivers/bus/auxiliary/version.map         |   7 +
 drivers/bus/meson.build                   |   1 +
 10 files changed, 921 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..eaf691ca6a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..e7ef4c8a05 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added auxiliary bus support.**
+
+  Auxiliary bus provides a way to split function into child-devices
+  representing sub-domains of functionality. Each auxiliary device
+  represents a part of its parent functionality.
+
 
 Removed Items
 -------------
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..8a75306da5
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,411 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the devices in the auxiliary bus.
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+/*
+ * Update a device's devargs being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary driver and device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((drv == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* Check if driver supports it. */
+	if (!auxiliary_match(drv, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	/* No initialization when marked as blocked, return without error. */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting socket to 0");
+		dev->device.numa_node = 0;
+	}
+
+	iova_mode = rte_eal_iova_mode();
+	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
+			      drv->driver.name);
+		return -EINVAL;
+	}
+
+	dev->driver = drv;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)",
+		      drv->driver.name, dev->name, dev->device.numa_node);
+	ret = drv->probe(drv, dev);
+	if (ret != 0)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &drv->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *drv;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	drv = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA socket %i",
+		      drv->driver.name, dev->name, dev->device.numa_node);
+
+	if (drv->remove != NULL) {
+		ret = drv->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *drv;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if (!drv->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(drv, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe function for
+ * all registered drivers to try to probe discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *drv = NULL;
+	const char **out = addr;
+
+	/* Allow empty device name "auxiliary:" to bypass entire bus scan. */
+	if (strlen(name) == 0)
+		return 0;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if (drv->match(name))
+			break;
+	}
+	if (drv != NULL && addr != NULL)
+		*out = name;
+	return drv != NULL ? 0 : -1;
+}
+
+/* Register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* Unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || aux_dev->driver == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map == NULL) {
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || aux_dev->driver == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap == NULL) {
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+}
+
+bool
+auxiliary_is_ignored_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..cd3fa56cb4
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			AUXILIARY_LOG(ERR, "cannot parse argument list %s",
+				      str);
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..8464487971
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get NUMA node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the devices in the auxiliary bus
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_is_ignored_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..357550eff7
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+headers = files(
+        'rte_bus_auxiliary.h',
+)
+sources = files(
+        'auxiliary_common.c',
+        'auxiliary_params.c',
+)
+if is_linux
+    sources += files(
+        'linux/auxiliary.c',
+    )
+endif
+deps += ['kvargs']
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..cb3e849993
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef AUXILIARY_PRIVATE_H
+#define AUXILIARY_PRIVATE_H
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, ...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, \
+		RTE_FMT("auxiliary bus: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
+			RTE_FMT_TAIL(__VA_ARGS__,)))
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARY_BUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARY_BUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+bool auxiliary_dev_exists(const char *name);
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list.
+ */
+int auxiliary_scan(void);
+
+/*
+ * Update a device being scanned.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ */
+bool auxiliary_is_ignored_device(const char *name);
+
+/*
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary device
+ * list). This function also updates the bus references of the auxiliary
+ * device and the generic device object embedded within.
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/*
+ * Match the auxiliary driver and device by driver function
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Iterate over devices, matching any device against the provided string
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* AUXILIARY_PRIVATE_H */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..16b147e387
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_BUS_AUXILIARY_H
+#define RTE_BUS_AUXILIARY_H
+
+/**
+ * @file
+ *
+ * Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+#define RTE_BUS_AUXILIARY_NAME "auxiliary"
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(rte_auxiliary_match_t)(const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(rte_auxiliary_probe_t)(struct rte_auxiliary_driver *drv,
+				    struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	struct rte_device device;                 /**< Inherit core device */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< Device driver */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;             /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;        /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t *match;         /**< Device match function. */
+	rte_auxiliary_probe_t *probe;         /**< Device probe function. */
+	rte_auxiliary_remove_t *remove;       /**< Device remove function. */
+	rte_auxiliary_dma_map_t *dma_map;     /**< Device DMA map function. */
+	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device DMA unmap function. */
+	uint32_t drv_flags;                   /**< Flags RTE_AUXILIARY_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv) \
+	RTE_INIT(auxiliaryinitfn_ ##nm) \
+	{ \
+		(auxiliary_drv).driver.name = RTE_STR(nm); \
+		rte_auxiliary_register(&(auxiliary_drv)); \
+	} \
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BUS_AUXILIARY_H */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..a52260657c
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,7 @@
+EXPERIMENTAL {
+	global:
+
+	# added in 21.08
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 410058de3a..45eab5233d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1

Re: [dpdk-dev] [PATCH v5 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

25/06/2021 05:26, Xueming(Steven) Li:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 23/06/2021 02:03, Xueming Li:
> > > +static int
> > > +auxiliary_parse(const char *name, void *addr) {
> > > +	struct rte_auxiliary_driver *drv = NULL;
> > > +	const char **out = addr;
> > > +
> > > +	/* Allow dummy name to prevent bus scan. */
> > > +	if (strlen(name) == 0)
> > > +		return 0;
> > 
> > Which syntax is it?
> 
> Allow empty device name "auxiliary:" to bypass entire auxiliary bus scan.

Ah OK
A suggestion of comment:
/* Skip auxiliary bus scan if name is empty. */

Re: [dpdk-dev] [PATCH v6 1/2] devargs: add common key definition

[Andrew Rybchenko]

On 6/25/21 2:47 PM, Xueming Li wrote:
> Adds common devargs key definition for "bus", "class" and "driver".
> 
> Acked-by: Thomas Monjalon <thomas@monjalon.net>
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>

Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Andrew Rybchenko]

On 6/25/21 2:47 PM, Xueming Li wrote:
> Auxiliary bus [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> Devargs legacy syntax ofauxiliary device:

Missing space after 'of'

>   -a auxiliary:<name>[,args...]
> Devargs generic syntax of auxiliary device:
>   -a bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,

Are two commas above intentionall? What for?

> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>

With my below notes fixed:

Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>

> Cc: Wang Haiyue <haiyue.wang@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>
> Cc: Kinsella Ray <mdr@ashroe.eu>
> ---
>  MAINTAINERS                               |   5 +
>  doc/guides/rel_notes/release_21_08.rst    |   6 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 411 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
>  drivers/bus/auxiliary/meson.build         |  16 +
>  drivers/bus/auxiliary/private.h           |  74 ++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
>  drivers/bus/auxiliary/version.map         |   7 +
>  drivers/bus/meson.build                   |   1 +
>  10 files changed, 921 insertions(+)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 5877a16971..eaf691ca6a 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
>  Bus Drivers
>  -----------
>  
> +Auxiliary bus driver

Shouldn't it be EXPERIMENTAL?

> +M: Parav Pandit <parav@nvidia.com>
> +M: Xueming Li <xuemingl@nvidia.com>
> +F: drivers/bus/auxiliary/
> +
>  Intel FPGA bus
>  M: Rosen Xu <rosen.xu@intel.com>
>  F: drivers/bus/ifpga/
> diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
> index a6ecfdf3ce..e7ef4c8a05 100644
> --- a/doc/guides/rel_notes/release_21_08.rst
> +++ b/doc/guides/rel_notes/release_21_08.rst
> @@ -55,6 +55,12 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
>  
> +* **Added auxiliary bus support.**
> +
> +  Auxiliary bus provides a way to split function into child-devices
> +  representing sub-domains of functionality. Each auxiliary device
> +  represents a part of its parent functionality.
> +
>  
>  Removed Items
>  -------------
> diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
> new file mode 100644
> index 0000000000..8a75306da5
> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
> @@ -0,0 +1,411 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> + */
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdbool.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <rte_errno.h>
> +#include <rte_interrupts.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_eal.h>
> +#include <rte_eal_paging.h>
> +#include <rte_string_fns.h>
> +#include <rte_common.h>
> +#include <rte_devargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name)
> +{
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
> +		if (strcmp(devargs->name, name) == 0)
> +			return devargs;
> +	}
> +	return NULL;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist

Missing full stop above.

> + *
> + * Stub for OS not supporting auxiliary bus.
> + */
> +__rte_weak bool
> +auxiliary_dev_exists(const char *name)
> +{
> +	RTE_SET_USED(name);
> +	return false;
> +}
> +
> +/*
> + * Scan the devices in the auxiliary bus.
> + *
> + * Stub for OS not supporting auxiliary bus.
> + */
> +__rte_weak int
> +auxiliary_scan(void)
> +{
> +	return 0;
> +}
> +
> +/*
> + * Update a device's devargs being scanned.
> + *
> + * @param aux_dev
> + *	AUXILIARY device.
> + */
> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
> +{
> +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> +}
> +
> +/*
> + * Match the auxiliary driver and device using driver function.
> + */
> +bool
> +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		const struct rte_auxiliary_device *aux_dev)
> +{
> +	if (aux_drv->match == NULL)
> +		return false;
> +	return aux_drv->match(aux_dev->name);
> +}
> +
> +/*
> + * Call the probe() function of the driver.
> + */
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
> +			       struct rte_auxiliary_device *dev)
> +{
> +	enum rte_iova_mode iova_mode;
> +	int ret;
> +
> +	if ((drv == NULL) || (dev == NULL))

Unnecessary internal parenthesis.

> +		return -EINVAL;
> +
> +	/* Check if driver supports it. */
> +	if (!auxiliary_match(drv, dev))
> +		/* Match of device and driver failed */
> +		return 1;
> +
> +	/* No initialization when marked as blocked, return without error. */
> +	if (dev->device.devargs != NULL &&
> +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> +		AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
> +		return -1;
> +	}
> +
> +	if (dev->device.numa_node < 0) {
> +		AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting socket to 0");

socket -> NUMA node

> +		dev->device.numa_node = 0;
> +	}
> +
> +	iova_mode = rte_eal_iova_mode();
> +	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> +	    iova_mode != RTE_IOVA_VA) {
> +		AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
> +			      drv->driver.name);
> +		return -EINVAL;
> +	}
> +
> +	dev->driver = drv;
> +
> +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)",

socket -> NUMA node

> +		      drv->driver.name, dev->name, dev->device.numa_node);
> +	ret = drv->probe(drv, dev);
> +	if (ret != 0)
> +		dev->driver = NULL;
> +	else
> +		dev->device.driver = &drv->driver;
> +
> +	return ret;
> +}
> +
> +/*
> + * Call the remove() function of the driver.
> + */
> +static int
> +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *drv;
> +	int ret = 0;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	drv = dev->driver;
> +
> +	AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA socket %i",

socket -> node

> +		      drv->driver.name, dev->name, dev->device.numa_node);
> +
> +	if (drv->remove != NULL) {
> +		ret = drv->remove(dev);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	/* clear driver structure */
> +	dev->driver = NULL;
> +	dev->device.driver = NULL;
> +
> +	return 0;
> +}
> +
> +/*
> + * Call the probe() function of all registered driver for the given device.
> + * Return < 0 if initialization failed.
> + * Return 1 if no driver is found for this device.
> + */
> +static int
> +auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
> +{
> +	struct rte_auxiliary_driver *drv;
> +	int rc;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> +		if (!drv->match(dev->name))
> +			continue;
> +
> +		rc = rte_auxiliary_probe_one_driver(drv, dev);
> +		if (rc < 0)
> +			/* negative value is an error */
> +			return rc;
> +		if (rc > 0)
> +			/* positive value means driver doesn't support it */
> +			continue;
> +		return 0;
> +	}
> +	return 1;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and call the probe function for
> + * all registered drivers to try to probe discovered devices.
> + */
> +static int
> +auxiliary_probe(void)
> +{
> +	struct rte_auxiliary_device *dev = NULL;
> +	size_t probed = 0, failed = 0;
> +	int ret = 0;
> +
> +	FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
> +		probed++;
> +
> +		ret = auxiliary_probe_all_drivers(dev);
> +		if (ret < 0) {
> +			if (ret != -EEXIST) {
> +				AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
> +					      dev->name);
> +				rte_errno = errno;
> +				failed++;
> +			}
> +			ret = 0;
> +		}
> +	}
> +
> +	return (probed && probed == failed) ? -1 : 0;
> +}
> +
> +static int
> +auxiliary_parse(const char *name, void *addr)
> +{
> +	struct rte_auxiliary_driver *drv = NULL;
> +	const char **out = addr;
> +
> +	/* Allow empty device name "auxiliary:" to bypass entire bus scan. */
> +	if (strlen(name) == 0)
> +		return 0;
> +
> +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> +		if (drv->match(name))
> +			break;
> +	}
> +	if (drv != NULL && addr != NULL)
> +		*out = name;
> +	return drv != NULL ? 0 : -1;
> +}
> +
> +/* Register a driver */
> +void
> +rte_auxiliary_register(struct rte_auxiliary_driver *driver)
> +{
> +	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
> +	driver->bus = &auxiliary_bus;
> +}
> +
> +/* Unregister a driver */
> +void
> +rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
> +{
> +	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
> +	driver->bus = NULL;
> +}
> +
> +/* Add a device to auxiliary bus */
> +void
> +auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
> +{
> +	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
> +}
> +
> +/* Insert a device into a predefined position in auxiliary bus */
> +void
> +auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			struct rte_auxiliary_device *new_aux_dev)
> +{
> +	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
> +}
> +
> +/* Remove a device from auxiliary bus */
> +static void
> +rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
> +{
> +	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
> +}
> +
> +static struct rte_device *
> +auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> +		      const void *data)
> +{
> +	const struct rte_auxiliary_device *pstart;
> +	struct rte_auxiliary_device *adev;
> +
> +	if (start != NULL) {
> +		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
> +		adev = TAILQ_NEXT(pstart, next);
> +	} else {
> +		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
> +	}
> +	while (adev != NULL) {
> +		if (cmp(&adev->device, data) == 0)
> +			return &adev->device;
> +		adev = TAILQ_NEXT(adev, next);
> +	}
> +	return NULL;
> +}
> +
> +static int
> +auxiliary_plug(struct rte_device *dev)
> +{
> +	if (!auxiliary_dev_exists(dev->name))
> +		return -ENOENT;
> +	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
> +}
> +
> +static int
> +auxiliary_unplug(struct rte_device *dev)
> +{
> +	struct rte_auxiliary_device *adev;
> +	int ret;
> +
> +	adev = RTE_DEV_TO_AUXILIARY(dev);
> +	ret = rte_auxiliary_driver_remove_dev(adev);
> +	if (ret == 0) {
> +		rte_auxiliary_remove_device(adev);
> +		rte_devargs_remove(dev->devargs);
> +		free(adev);
> +	}
> +	return ret;
> +}
> +
> +static int
> +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> +{
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || aux_dev->driver == NULL) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_map == NULL) {
> +		rte_errno = ENOTSUP;
> +		return -1;
> +	}
> +	return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
> +}
> +
> +static int
> +auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
> +		    size_t len)
> +{
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || aux_dev->driver == NULL) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_unmap == NULL) {
> +		rte_errno = ENOTSUP;
> +		return -1;
> +	}
> +	return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
> +}
> +
> +bool
> +auxiliary_is_ignored_device(const char *name)
> +{
> +	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
> +
> +	switch (auxiliary_bus.bus.conf.scan_mode) {
> +	case RTE_BUS_SCAN_ALLOWLIST:
> +		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
> +			return false;
> +		break;
> +	case RTE_BUS_SCAN_UNDEFINED:
> +	case RTE_BUS_SCAN_BLOCKLIST:
> +		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
> +			return false;
> +		break;
> +	}
> +	return true;
> +}
> +
> +static enum rte_iova_mode
> +auxiliary_get_iommu_class(void)
> +{
> +	const struct rte_auxiliary_driver *drv;
> +
> +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> +		if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
> +			return RTE_IOVA_VA;
> +	}
> +
> +	return RTE_IOVA_DC;
> +}
> +
> +struct rte_auxiliary_bus auxiliary_bus = {
> +	.bus = {
> +		.scan = auxiliary_scan,
> +		.probe = auxiliary_probe,
> +		.find_device = auxiliary_find_device,
> +		.plug = auxiliary_plug,
> +		.unplug = auxiliary_unplug,
> +		.parse = auxiliary_parse,
> +		.dma_map = auxiliary_dma_map,
> +		.dma_unmap = auxiliary_dma_unmap,
> +		.get_iommu_class = auxiliary_get_iommu_class,
> +		.dev_iterate = auxiliary_dev_iterate,
> +	},
> +	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
> +	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
> +};
> +
> +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> +RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
> diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
> new file mode 100644
> index 0000000000..cd3fa56cb4
> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_params.c
> @@ -0,0 +1,59 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> + */
> +
> +#include <string.h>
> +
> +#include <rte_bus.h>
> +#include <rte_dev.h>
> +#include <rte_errno.h>
> +#include <rte_kvargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +enum auxiliary_params {
> +	RTE_AUXILIARY_PARAM_NAME,
> +};
> +
> +static const char * const auxiliary_params_keys[] = {
> +	[RTE_AUXILIARY_PARAM_NAME] = "name",
> +};
> +
> +static int
> +auxiliary_dev_match(const struct rte_device *dev,
> +	      const void *_kvlist)
> +{
> +	const struct rte_kvargs *kvlist = _kvlist;
> +	int ret;
> +
> +	ret = rte_kvargs_process(kvlist,
> +			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
> +			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
> +
> +	return ret != 0 ? -1 : 0;
> +}
> +
> +void *
> +auxiliary_dev_iterate(const void *start,
> +		    const char *str,
> +		    const struct rte_dev_iterator *it __rte_unused)
> +{
> +	rte_bus_find_device_t find_device;
> +	struct rte_kvargs *kvargs = NULL;
> +	struct rte_device *dev;
> +
> +	if (str != NULL) {
> +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> +		if (kvargs == NULL) {
> +			AUXILIARY_LOG(ERR, "cannot parse argument list %s",
> +				      str);
> +			rte_errno = EINVAL;
> +			return NULL;
> +		}
> +	}
> +	find_device = auxiliary_bus.bus.find_device;
> +	dev = find_device(start, auxiliary_dev_match, kvargs);
> +	rte_kvargs_free(kvargs);
> +	return dev;
> +}
> diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
> new file mode 100644
> index 0000000000..8464487971
> --- /dev/null
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> @@ -0,0 +1,141 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> + */
> +
> +#include <string.h>
> +#include <dirent.h>
> +
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <eal_filesystem.h>
> +
> +#include "../rte_bus_auxiliary.h"
> +#include "../private.h"
> +
> +#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
> +
> +/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
> +static int
> +auxiliary_scan_one(const char *dirname, const char *name)
> +{
> +	struct rte_auxiliary_device *dev;
> +	struct rte_auxiliary_device *dev2;
> +	char filename[PATH_MAX];
> +	unsigned long tmp;
> +	int ret;
> +
> +	dev = malloc(sizeof(*dev));
> +	if (dev == NULL)
> +		return -1;
> +
> +	memset(dev, 0, sizeof(*dev));
> +	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
> +		free(dev);
> +		return -1;
> +	}
> +	dev->device.name = dev->name;
> +	dev->device.bus = &auxiliary_bus.bus;
> +
> +	/* Get NUMA node, default to 0 if not present */
> +	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
> +		 dirname, name);
> +	if (access(filename, F_OK) != -1) {
> +		if (eal_parse_sysfs_value(filename, &tmp) == 0)
> +			dev->device.numa_node = tmp;
> +		else
> +			dev->device.numa_node = -1;
> +	} else {
> +		dev->device.numa_node = 0;
> +	}
> +
> +	auxiliary_on_scan(dev);
> +
> +	/* Device is valid, add in list (sorted) */
> +	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
> +		ret = strcmp(dev->name, dev2->name);
> +		if (ret > 0)
> +			continue;
> +		if (ret < 0) {
> +			auxiliary_insert_device(dev2, dev);
> +		} else { /* already registered */
> +			if (rte_dev_is_probed(&dev2->device) &&
> +			    dev2->device.devargs != dev->device.devargs) {
> +				/* To probe device with new devargs. */
> +				rte_devargs_remove(dev2->device.devargs);
> +				auxiliary_on_scan(dev2);
> +			}
> +			free(dev);
> +		}
> +		return 0;
> +	}
> +	auxiliary_add_device(dev);
> +	return 0;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist

Missing full stop above.

> + */
> +bool
> +auxiliary_dev_exists(const char *name)
> +{
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +
> +	snprintf(dirname, sizeof(dirname), "%s/%s",
> +		 AUXILIARY_SYSFS_PATH, name);
> +	dir = opendir(dirname);
> +	if (dir == NULL)
> +		return false;
> +	closedir(dir);
> +	return true;
> +}
> +
> +/*
> + * Scan the devices in the auxiliary bus

Missing full stop above.

> + */
> +int
> +auxiliary_scan(void)
> +{
> +	struct dirent *e;
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +	struct rte_auxiliary_driver *drv;
> +
> +	dir = opendir(AUXILIARY_SYSFS_PATH);
> +	if (dir == NULL) {
> +		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?",
> +			      AUXILIARY_SYSFS_PATH);
> +		return 0;
> +	}
> +
> +	while ((e = readdir(dir)) != NULL) {
> +		if (e->d_name[0] == '.')
> +			continue;
> +
> +		if (auxiliary_is_ignored_device(e->d_name))
> +			continue;
> +
> +		snprintf(dirname, sizeof(dirname), "%s/%s",
> +			 AUXILIARY_SYSFS_PATH, e->d_name);
> +
> +		/* Ignore if no driver can handle. */
> +		FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> +			if (drv->match(e->d_name))
> +				break;
> +		}
> +		if (drv == NULL)
> +			continue;
> +
> +		if (auxiliary_scan_one(dirname, e->d_name) < 0)
> +			goto error;
> +	}
> +	closedir(dir);
> +	return 0;
> +
> +error:
> +	closedir(dir);
> +	return -1;
> +}
> diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
> new file mode 100644
> index 0000000000..357550eff7
> --- /dev/null
> +++ b/drivers/bus/auxiliary/meson.build
> @@ -0,0 +1,16 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright (c) 2021 NVIDIA Corporation & Affiliates
> +
> +headers = files(
> +        'rte_bus_auxiliary.h',
> +)
> +sources = files(
> +        'auxiliary_common.c',
> +        'auxiliary_params.c',
> +)
> +if is_linux
> +    sources += files(
> +        'linux/auxiliary.c',
> +    )
> +endif
> +deps += ['kvargs']
> diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
> new file mode 100644
> index 0000000000..cb3e849993
> --- /dev/null
> +++ b/drivers/bus/auxiliary/private.h
> @@ -0,0 +1,74 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> + */
> +
> +#ifndef AUXILIARY_PRIVATE_H

May be add BUS_ prefix at leaat?

> +#define AUXILIARY_PRIVATE_H
> +
> +#include <stdbool.h>
> +#include <stdio.h>
> +
> +#include "rte_bus_auxiliary.h"
> +
> +extern struct rte_auxiliary_bus auxiliary_bus;
> +extern int auxiliary_bus_logtype;
> +
> +#define AUXILIARY_LOG(level, ...) \
> +	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, \
> +		RTE_FMT("auxiliary bus: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
> +			RTE_FMT_TAIL(__VA_ARGS__,)))
> +
> +/* Auxiliary bus iterators */
> +#define FOREACH_DEVICE_ON_AUXILIARY_BUS(p) \
> +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> +
> +#define FOREACH_DRIVER_ON_AUXILIARY_BUS(p) \
> +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> +
> +bool auxiliary_dev_exists(const char *name);
> +
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the devices
> + * list.
> + */
> +int auxiliary_scan(void);
> +
> +/*
> + * Update a device being scanned.
> + */
> +void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
> +
> +/*
> + * Validate whether a device with given auxiliary device should be ignored
> + * or not.
> + */
> +bool auxiliary_is_ignored_device(const char *name);
> +
> +/*
> + * Add an auxiliary device to the auxiliary bus (append to auxiliary device
> + * list). This function also updates the bus references of the auxiliary
> + * device and the generic device object embedded within.
> + */
> +void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
> +
> +/*
> + * Insert an auxiliary device in the auxiliary bus at a particular location
> + * in the device list. It also updates the auxiliary bus reference of the
> + * new devices to be inserted.
> + */
> +void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			     struct rte_auxiliary_device *new_aux_dev);
> +
> +/*
> + * Match the auxiliary driver and device by driver function

Missing full stop.

> + */
> +bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		     const struct rte_auxiliary_device *aux_dev);
> +
> +/*
> + * Iterate over devices, matching any device against the provided string

Missing full stop.

> + */
> +void *auxiliary_dev_iterate(const void *start, const char *str,
> +			    const struct rte_dev_iterator *it);
> +
> +#endif /* AUXILIARY_PRIVATE_H */
> diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> new file mode 100644
> index 0000000000..16b147e387
> --- /dev/null
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> @@ -0,0 +1,201 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> + */
> +
> +#ifndef RTE_BUS_AUXILIARY_H
> +#define RTE_BUS_AUXILIARY_H
> +
> +/**
> + * @file
> + *
> + * Auxiliary Bus Interface.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <sys/queue.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +
> +#include <rte_debug.h>
> +#include <rte_interrupts.h>
> +#include <rte_dev.h>
> +#include <rte_bus.h>
> +#include <rte_kvargs.h>
> +
> +#define RTE_BUS_AUXILIARY_NAME "auxiliary"
> +
> +/* Forward declarations */
> +struct rte_auxiliary_driver;
> +struct rte_auxiliary_bus;
> +struct rte_auxiliary_device;
> +
> +/**
> + * Match function for the driver to decide if device can be handled.
> + *
> + * @param name
> + *   Pointer to the auxiliary device name.
> + * @return
> + *   Whether the driver can handle the auxiliary device.
> + */
> +typedef bool(rte_auxiliary_match_t)(const char *name);
> +
> +/**
> + * Initialization function for the driver called during auxiliary probing.
> + *
> + * @param drv
> + *   Pointer to the auxiliary driver.
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int(rte_auxiliary_probe_t)(struct rte_auxiliary_driver *drv,
> +				    struct rte_auxiliary_device *dev);
> +
> +/**
> + * Uninitialization function for the driver called during hotplugging.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
> +
> +/**
> + * Driver-specific DMA mapping. After a successful call the device
> + * will be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be mapped.
> + * @param iova
> + *   Starting IOVA address of memory to be mapped.
> + * @param len
> + *   Length of memory segment being mapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
> +				       void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * Driver-specific DMA un-mapping. After a successful call the device
> + * will not be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be unmapped.
> + * @param iova
> + *   Starting IOVA address of memory to be unmapped.
> + * @param len
> + *   Length of memory segment being unmapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> +					 void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * A structure describing an auxiliary device.
> + */
> +struct rte_auxiliary_device {
> +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> +	struct rte_device device;                 /**< Inherit core device */
> +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> +	struct rte_auxiliary_driver *driver;      /**< Device driver */
> +};
> +
> +/** List of auxiliary devices */
> +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> +/** List of auxiliary drivers */
> +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);

Shouldn't we hide rte_auxiliary_device inside the library take
API/ABI stability into account? Or will be it DPDK internal anyway? If
so, it should be done INTERNAL from the very
beginning.

> +
> +/**
> + * Structure describing the auxiliary bus
> + */
> +struct rte_auxiliary_bus {
> +	struct rte_bus bus;                  /**< Inherit the generic class */
> +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
> +};

It looks internal. The following forward declaration should be
sufficient to build.

struct rte_auxiliary_bus;


> +
> +/**
> + * A structure describing an auxiliary driver.
> + */
> +struct rte_auxiliary_driver {
> +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> +	struct rte_driver driver;             /**< Inherit core driver. */
> +	struct rte_auxiliary_bus *bus;        /**< Auxiliary bus reference. */
> +	rte_auxiliary_match_t *match;         /**< Device match function. */
> +	rte_auxiliary_probe_t *probe;         /**< Device probe function. */
> +	rte_auxiliary_remove_t *remove;       /**< Device remove function. */
> +	rte_auxiliary_dma_map_t *dma_map;     /**< Device DMA map function. */
> +	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device DMA unmap function. */
> +	uint32_t drv_flags;                   /**< Flags RTE_AUXILIARY_DRV_*. */
> +};
> +
> +/**
> + * @internal
> + * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
> + */
> +#define RTE_DEV_TO_AUXILIARY(ptr) \
> +	container_of(ptr, struct rte_auxiliary_device, device)
> +
> +#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
> +	container_of(ptr, const struct rte_auxiliary_device, device)
> +
> +#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
> +	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
> +
> +/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
> +#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
> +
> +/**

Don't we need EXPERIMENTAL notice here?

> + * Register an auxiliary driver.
> + *
> + * @param driver
> + *   A pointer to a rte_auxiliary_driver structure describing the driver
> + *   to be registered.
> + */
> +__rte_experimental
> +void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
> +
> +/** Helper for auxiliary device registration from driver instance */
> +#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv) \
> +	RTE_INIT(auxiliaryinitfn_ ##nm) \
> +	{ \
> +		(auxiliary_drv).driver.name = RTE_STR(nm); \
> +		rte_auxiliary_register(&(auxiliary_drv)); \
> +	} \
> +	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
> +
> +/**

Don't we need EXPERIMENTAL notice here?

> + * Unregister an auxiliary driver.
> + *
> + * @param driver
> + *   A pointer to a rte_auxiliary_driver structure describing the driver
> + *   to be unregistered.
> + */
> +__rte_experimental
> +void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* RTE_BUS_AUXILIARY_H */
> diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
> new file mode 100644
> index 0000000000..a52260657c
> --- /dev/null
> +++ b/drivers/bus/auxiliary/version.map
> @@ -0,0 +1,7 @@
> +EXPERIMENTAL {
> +	global:
> +
> +	# added in 21.08
> +	rte_auxiliary_register;
> +	rte_auxiliary_unregister;
> +};
> diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
> index 410058de3a..45eab5233d 100644
> --- a/drivers/bus/meson.build
> +++ b/drivers/bus/meson.build
> @@ -2,6 +2,7 @@
>  # Copyright(c) 2017 Intel Corporation
>  
>  drivers = [
> +        'auxiliary',
>          'dpaa',
>          'fslmc',
>          'ifpga',
> 

[dpdk-dev] [PATCH v7 1/2] devargs: add common key definition

[Xueming Li]

Adds common devargs key definition for "bus", "class" and "driver".

Acked-by: Thomas Monjalon <thomas@monjalon.net>
Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |  2 --
 drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
 drivers/common/sfc_efx/sfc_efx.c        |  7 +++----
 drivers/common/sfc_efx/sfc_efx.h        |  2 --
 drivers/net/bonding/rte_eth_bond_args.c |  2 +-
 drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
 drivers/net/iavf/iavf_ethdev.c          |  5 ++---
 drivers/net/mlx5/mlx5.c                 |  4 ++--
 drivers/net/sfc/sfc_kvargs.c            |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
 lib/eal/common/eal_common_devargs.c     | 12 ++++++------
 lib/eal/include/rte_devargs.h           | 24 ++++++++++++++++++++++++
 12 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..306f2f1ab7 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -208,8 +208,6 @@ __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
 
-#define MLX5_CLASS_ARG_NAME "class"
-
 enum mlx5_class {
 	MLX5_CLASS_INVALID,
 	MLX5_CLASS_NET = RTE_BIT64(0),
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..34747c4e07 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,
 static int
 parse_class_options(const struct rte_devargs *devargs)
 {
-	const char *key = MLX5_CLASS_ARG_NAME;
+	const char *key = RTE_DEVARGS_KEY_CLASS;
 	struct rte_kvargs *kvlist;
 	int ret = 0;
 
diff --git a/drivers/common/sfc_efx/sfc_efx.c b/drivers/common/sfc_efx/sfc_efx.c
index 0b78933d9f..2dc5545760 100644
--- a/drivers/common/sfc_efx/sfc_efx.c
+++ b/drivers/common/sfc_efx/sfc_efx.c
@@ -42,7 +42,6 @@ enum sfc_efx_dev_class
 sfc_efx_dev_class_get(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvargs;
-	const char *key = SFC_EFX_KVARG_DEV_CLASS;
 	enum sfc_efx_dev_class dev_class = SFC_EFX_DEV_CLASS_NET;
 
 	if (devargs == NULL)
@@ -52,9 +51,9 @@ sfc_efx_dev_class_get(struct rte_devargs *devargs)
 	if (kvargs == NULL)
 		return dev_class;
 
-	if (rte_kvargs_count(kvargs, key) != 0) {
-		rte_kvargs_process(kvargs, key, sfc_efx_kvarg_dev_class_handler,
-				   &dev_class);
+	if (rte_kvargs_count(kvargs, RTE_DEVARGS_KEY_CLASS) != 0) {
+		rte_kvargs_process(kvargs, RTE_DEVARGS_KEY_CLASS,
+				   sfc_efx_kvarg_dev_class_handler, &dev_class);
 	}
 
 	rte_kvargs_free(kvargs);
diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
index 6b6164cb1f..c16eca60f3 100644
--- a/drivers/common/sfc_efx/sfc_efx.h
+++ b/drivers/common/sfc_efx/sfc_efx.h
@@ -19,8 +19,6 @@
 extern "C" {
 #endif
 
-#define SFC_EFX_KVARG_DEV_CLASS	"class"
-
 enum sfc_efx_dev_class {
 	SFC_EFX_DEV_CLASS_INVALID = 0,
 	SFC_EFX_DEV_CLASS_NET,
diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
index 764b1b8c8e..5406e1c934 100644
--- a/drivers/net/bonding/rte_eth_bond_args.c
+++ b/drivers/net/bonding/rte_eth_bond_args.c
@@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
 	PMD_BOND_SOCKET_ID_KVARG,
 	PMD_BOND_MAC_ADDR_KVARG,
 	PMD_BOND_AGG_MODE_KVARG,
-	"driver",
+	RTE_DEVARGS_KEY_DRIVER,
 	NULL
 };
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 385ebedcd3..0cfe13b7b2 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1660,7 +1660,6 @@ static int
 i40evf_driver_selected(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (devargs == NULL)
@@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       i40evf_check_driver_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 5290588b17..472538181e 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -2448,7 +2448,6 @@ static int
 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (device_id != IAVF_DEV_ID_VF &&
@@ -2464,13 +2463,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       iavf_drv_i40evf_check_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..d0faa45944 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->max_dump_files_num = tmp;
 	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
 		config->lro.timeout = tmp;
-	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
+	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
 		DRV_LOG(DEBUG, "class argument is %s.", val);
 	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
 		config->log_hp_size = tmp;
@@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_REPRESENTOR,
 		MLX5_MAX_DUMP_FILES_NUM,
 		MLX5_LRO_TIMEOUT_USEC,
-		MLX5_CLASS_ARG_NAME,
+		RTE_DEVARGS_KEY_CLASS,
 		MLX5_HP_BUF_SIZE,
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 0efa92ed28..974c05e68e 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
 		SFC_KVARG_TX_DATAPATH,
 		SFC_KVARG_FW_VARIANT,
 		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
-		SFC_EFX_KVARG_DEV_CLASS,
+		RTE_DEVARGS_KEY_CLASS,
 		NULL,
 	};
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index e5e03e6582..8b5bfd8c3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 	unsigned long tmp;
 	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 
-	if (strcmp(key, "class") == 0)
+	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
 		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
index b31ac879a9..23aaf8b7e4 100644
--- a/lib/eal/common/eal_common_devargs.c
+++ b/lib/eal/common/eal_common_devargs.c
@@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		const char *str;
 		struct rte_kvargs *kvlist;
 	} layers[] = {
-		{ "bus=",    NULL, NULL, },
-		{ "class=",  NULL, NULL, },
-		{ "driver=", NULL, NULL, },
+		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
+		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
+		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
 	};
 	struct rte_kvargs_pair *kv = NULL;
 	struct rte_class *cls = NULL;
@@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		if (layers[i].kvlist == NULL)
 			continue;
 		kv = &layers[i].kvlist->pairs[0];
-		if (strcmp(kv->key, "bus") == 0) {
+		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
 			bus = rte_bus_find_by_name(kv->value);
 			if (bus == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
@@ -126,7 +126,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "class") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
 			cls = rte_class_find_by_name(kv->value);
 			if (cls == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
@@ -134,7 +134,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "driver") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
 			/* Ignore */
 			continue;
 		}
diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h
index 1e595b3c51..cd90944fe8 100644
--- a/lib/eal/include/rte_devargs.h
+++ b/lib/eal/include/rte_devargs.h
@@ -25,6 +25,30 @@ extern "C" {
 #include <rte_compat.h>
 #include <rte_bus.h>
 
+/**
+ * Bus type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't use this key as bus type
+ * is resolved as first optional value separated by ":".
+ */
+#define RTE_DEVARGS_KEY_BUS "bus"
+
+/**
+ * Class type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't parse class type. PMD driver is
+ * encouraged to use this key to resolve class type.
+ */
+#define RTE_DEVARGS_KEY_CLASS "class"
+
+/**
+ * Driver type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't parse driver type. PMD driver is
+ * encouraged to use this key to resolve driver type.
+ */
+#define RTE_DEVARGS_KEY_DRIVER "driver"
+
 /**
  * Type of generic device
  */
-- 
2.25.1

[dpdk-dev] [PATCH v7 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary bus [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

Devargs legacy syntax of auxiliary device:
  -a auxiliary:<name>[,args...]
Devargs generic syntax of auxiliary device:
  -a bus=auxiliary,name=<name>/class=<class>/driver=<driver>[,args...]

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
Cc: Thomas Monjalon <thomas@monjalon.net>
Cc: Kinsella Ray <mdr@ashroe.eu>
Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
---
 MAINTAINERS                               |   5 +
 doc/guides/rel_notes/release_21_08.rst    |   6 +
 drivers/bus/auxiliary/auxiliary_common.c  | 411 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
 drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
 drivers/bus/auxiliary/meson.build         |  16 +
 drivers/bus/auxiliary/private.h           |  74 ++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
 drivers/bus/auxiliary/version.map         |   7 +
 drivers/bus/meson.build                   |   1 +
 10 files changed, 921 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..9b0d0d4348 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver - EXPERIMENTAL
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..e7ef4c8a05 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added auxiliary bus support.**
+
+  Auxiliary bus provides a way to split function into child-devices
+  representing sub-domains of functionality. Each auxiliary device
+  represents a part of its parent functionality.
+
 
 Removed Items
 -------------
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..8a75306da5
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,411 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the devices in the auxiliary bus.
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+/*
+ * Update a device's devargs being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary driver and device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if ((drv == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	/* Check if driver supports it. */
+	if (!auxiliary_match(drv, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	/* No initialization when marked as blocked, return without error. */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting socket to 0");
+		dev->device.numa_node = 0;
+	}
+
+	iova_mode = rte_eal_iova_mode();
+	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
+			      drv->driver.name);
+		return -EINVAL;
+	}
+
+	dev->driver = drv;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)",
+		      drv->driver.name, dev->name, dev->device.numa_node);
+	ret = drv->probe(drv, dev);
+	if (ret != 0)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &drv->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *drv;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	drv = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA socket %i",
+		      drv->driver.name, dev->name, dev->device.numa_node);
+
+	if (drv->remove != NULL) {
+		ret = drv->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *drv;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if (!drv->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(drv, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe function for
+ * all registered drivers to try to probe discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *drv = NULL;
+	const char **out = addr;
+
+	/* Allow empty device name "auxiliary:" to bypass entire bus scan. */
+	if (strlen(name) == 0)
+		return 0;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if (drv->match(name))
+			break;
+	}
+	if (drv != NULL && addr != NULL)
+		*out = name;
+	return drv != NULL ? 0 : -1;
+}
+
+/* Register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* Unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || aux_dev->driver == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map == NULL) {
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || aux_dev->driver == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap == NULL) {
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+}
+
+bool
+auxiliary_is_ignored_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..cd3fa56cb4
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			AUXILIARY_LOG(ERR, "cannot parse argument list %s",
+				      str);
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..8464487971
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get NUMA node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the devices in the auxiliary bus
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_is_ignored_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..357550eff7
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+headers = files(
+        'rte_bus_auxiliary.h',
+)
+sources = files(
+        'auxiliary_common.c',
+        'auxiliary_params.c',
+)
+if is_linux
+    sources += files(
+        'linux/auxiliary.c',
+    )
+endif
+deps += ['kvargs']
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..cb3e849993
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef AUXILIARY_PRIVATE_H
+#define AUXILIARY_PRIVATE_H
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, ...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, \
+		RTE_FMT("auxiliary bus: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
+			RTE_FMT_TAIL(__VA_ARGS__,)))
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARY_BUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARY_BUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+bool auxiliary_dev_exists(const char *name);
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list.
+ */
+int auxiliary_scan(void);
+
+/*
+ * Update a device being scanned.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ */
+bool auxiliary_is_ignored_device(const char *name);
+
+/*
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary device
+ * list). This function also updates the bus references of the auxiliary
+ * device and the generic device object embedded within.
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/*
+ * Match the auxiliary driver and device by driver function
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Iterate over devices, matching any device against the provided string
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* AUXILIARY_PRIVATE_H */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..16b147e387
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_BUS_AUXILIARY_H
+#define RTE_BUS_AUXILIARY_H
+
+/**
+ * @file
+ *
+ * Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+#define RTE_BUS_AUXILIARY_NAME "auxiliary"
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(rte_auxiliary_match_t)(const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(rte_auxiliary_probe_t)(struct rte_auxiliary_driver *drv,
+				    struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	struct rte_device device;                 /**< Inherit core device */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< Device driver */
+};
+
+/** List of auxiliary devices */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/** List of auxiliary drivers */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/**
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /**< Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /**< List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;             /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;        /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t *match;         /**< Device match function. */
+	rte_auxiliary_probe_t *probe;         /**< Device probe function. */
+	rte_auxiliary_remove_t *remove;       /**< Device remove function. */
+	rte_auxiliary_dma_map_t *dma_map;     /**< Device DMA map function. */
+	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device DMA unmap function. */
+	uint32_t drv_flags;                   /**< Flags RTE_AUXILIARY_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv) \
+	RTE_INIT(auxiliaryinitfn_ ##nm) \
+	{ \
+		(auxiliary_drv).driver.name = RTE_STR(nm); \
+		rte_auxiliary_register(&(auxiliary_drv)); \
+	} \
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BUS_AUXILIARY_H */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..a52260657c
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,7 @@
+EXPERIMENTAL {
+	global:
+
+	# added in 21.08
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 410058de3a..45eab5233d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

Hi Andrew,

Thanks very much all the good suggestions, v7 posted.

Best Regards,
Xueming

> -----Original Message-----
> From: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> Sent: Monday, July 5, 2021 12:13 AM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Kinsella Ray
> <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Subject: Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus
> 
> On 6/25/21 2:47 PM, Xueming Li wrote:
> > Auxiliary bus [1] provides a way to split function into child-devices
> > representing sub-domains of functionality. Each auxiliary device
> > represents a part of its parent functionality.
> >
> > Auxiliary device is identified by unique device name, sysfs path:
> >   /sys/bus/auxiliary/devices/<name>
> >
> > Devargs legacy syntax ofauxiliary device:
> 
> Missing space after 'of'
> 
> >   -a auxiliary:<name>[,args...]
> > Devargs generic syntax of auxiliary device:
> >   -a bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,
> 
> Are two commas above intentionall? What for?
> 
> >
> > [1] kernel auxiliary bus document:
> > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> 
> With my below notes fixed:
> 
> Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> 
> > Cc: Wang Haiyue <haiyue.wang@intel.com>
> > Cc: Thomas Monjalon <thomas@monjalon.net>
> > Cc: Kinsella Ray <mdr@ashroe.eu>
> > ---
> >  MAINTAINERS                               |   5 +
> >  doc/guides/rel_notes/release_21_08.rst    |   6 +
> >  drivers/bus/auxiliary/auxiliary_common.c  | 411
> > ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
> >  drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
> >  drivers/bus/auxiliary/meson.build         |  16 +
> >  drivers/bus/auxiliary/private.h           |  74 ++++
> >  drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
> >  drivers/bus/auxiliary/version.map         |   7 +
> >  drivers/bus/meson.build                   |   1 +
> >  10 files changed, 921 insertions(+)
> >  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
> >  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> >  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> >  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> > 100644 drivers/bus/auxiliary/private.h  create mode 100644
> > drivers/bus/auxiliary/rte_bus_auxiliary.h
> >  create mode 100644 drivers/bus/auxiliary/version.map
> >
> > diff --git a/MAINTAINERS b/MAINTAINERS index 5877a16971..eaf691ca6a
> > 100644
> > --- a/MAINTAINERS
> > +++ b/MAINTAINERS
> > @@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst  Bus Drivers
> >  -----------
> >
> > +Auxiliary bus driver
> 
> Shouldn't it be EXPERIMENTAL?
> 
> > +M: Parav Pandit <parav@nvidia.com>
> > +M: Xueming Li <xuemingl@nvidia.com>
> > +F: drivers/bus/auxiliary/
> > +
> >  Intel FPGA bus
> >  M: Rosen Xu <rosen.xu@intel.com>
> >  F: drivers/bus/ifpga/
> > diff --git a/doc/guides/rel_notes/release_21_08.rst
> > b/doc/guides/rel_notes/release_21_08.rst
> > index a6ecfdf3ce..e7ef4c8a05 100644
> > --- a/doc/guides/rel_notes/release_21_08.rst
> > +++ b/doc/guides/rel_notes/release_21_08.rst
> > @@ -55,6 +55,12 @@ New Features
> >       Also, make sure to start the actual text at the margin.
> >       =======================================================
> >
> > +* **Added auxiliary bus support.**
> > +
> > +  Auxiliary bus provides a way to split function into child-devices
> > + representing sub-domains of functionality. Each auxiliary device
> > + represents a part of its parent functionality.
> > +
> >
> >  Removed Items
> >  -------------
> > diff --git a/drivers/bus/auxiliary/auxiliary_common.c
> > b/drivers/bus/auxiliary/auxiliary_common.c
> > new file mode 100644
> > index 0000000000..8a75306da5
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/auxiliary_common.c
> > @@ -0,0 +1,411 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright (c) 2021 NVIDIA Corporation & Affiliates  */
> > +
> > +#include <string.h>
> > +#include <inttypes.h>
> > +#include <stdint.h>
> > +#include <stdbool.h>
> > +#include <stdlib.h>
> > +#include <stdio.h>
> > +#include <sys/queue.h>
> > +#include <rte_errno.h>
> > +#include <rte_interrupts.h>
> > +#include <rte_log.h>
> > +#include <rte_bus.h>
> > +#include <rte_per_lcore.h>
> > +#include <rte_memory.h>
> > +#include <rte_eal.h>
> > +#include <rte_eal_paging.h>
> > +#include <rte_string_fns.h>
> > +#include <rte_common.h>
> > +#include <rte_devargs.h>
> > +
> > +#include "private.h"
> > +#include "rte_bus_auxiliary.h"
> > +
> > +static struct rte_devargs *
> > +auxiliary_devargs_lookup(const char *name) {
> > +	struct rte_devargs *devargs;
> > +
> > +	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
> > +		if (strcmp(devargs->name, name) == 0)
> > +			return devargs;
> > +	}
> > +	return NULL;
> > +}
> > +
> > +/*
> > + * Test whether the auxiliary device exist
> 
> Missing full stop above.
> 
> > + *
> > + * Stub for OS not supporting auxiliary bus.
> > + */
> > +__rte_weak bool
> > +auxiliary_dev_exists(const char *name) {
> > +	RTE_SET_USED(name);
> > +	return false;
> > +}
> > +
> > +/*
> > + * Scan the devices in the auxiliary bus.
> > + *
> > + * Stub for OS not supporting auxiliary bus.
> > + */
> > +__rte_weak int
> > +auxiliary_scan(void)
> > +{
> > +	return 0;
> > +}
> > +
> > +/*
> > + * Update a device's devargs being scanned.
> > + *
> > + * @param aux_dev
> > + *	AUXILIARY device.
> > + */
> > +void
> > +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev) {
> > +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> > +}
> > +
> > +/*
> > + * Match the auxiliary driver and device using driver function.
> > + */
> > +bool
> > +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> > +		const struct rte_auxiliary_device *aux_dev) {
> > +	if (aux_drv->match == NULL)
> > +		return false;
> > +	return aux_drv->match(aux_dev->name); }
> > +
> > +/*
> > + * Call the probe() function of the driver.
> > + */
> > +static int
> > +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
> > +			       struct rte_auxiliary_device *dev)
> > +{
> > +	enum rte_iova_mode iova_mode;
> > +	int ret;
> > +
> > +	if ((drv == NULL) || (dev == NULL))
> 
> Unnecessary internal parenthesis.
> 
> > +		return -EINVAL;
> > +
> > +	/* Check if driver supports it. */
> > +	if (!auxiliary_match(drv, dev))
> > +		/* Match of device and driver failed */
> > +		return 1;
> > +
> > +	/* No initialization when marked as blocked, return without error. */
> > +	if (dev->device.devargs != NULL &&
> > +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> > +		AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
> > +		return -1;
> > +	}
> > +
> > +	if (dev->device.numa_node < 0) {
> > +		AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting socket to 0");
> 
> socket -> NUMA node
> 
> > +		dev->device.numa_node = 0;
> > +	}
> > +
> > +	iova_mode = rte_eal_iova_mode();
> > +	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> > +	    iova_mode != RTE_IOVA_VA) {
> > +		AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
> > +			      drv->driver.name);
> > +		return -EINVAL;
> > +	}
> > +
> > +	dev->driver = drv;
> > +
> > +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)",
> 
> socket -> NUMA node
> 
> > +		      drv->driver.name, dev->name, dev->device.numa_node);
> > +	ret = drv->probe(drv, dev);
> > +	if (ret != 0)
> > +		dev->driver = NULL;
> > +	else
> > +		dev->device.driver = &drv->driver;
> > +
> > +	return ret;
> > +}
> > +
> > +/*
> > + * Call the remove() function of the driver.
> > + */
> > +static int
> > +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
> > +{
> > +	struct rte_auxiliary_driver *drv;
> > +	int ret = 0;
> > +
> > +	if (dev == NULL)
> > +		return -EINVAL;
> > +
> > +	drv = dev->driver;
> > +
> > +	AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA socket %i",
> 
> socket -> node
> 
> > +		      drv->driver.name, dev->name, dev->device.numa_node);
> > +
> > +	if (drv->remove != NULL) {
> > +		ret = drv->remove(dev);
> > +		if (ret < 0)
> > +			return ret;
> > +	}
> > +
> > +	/* clear driver structure */
> > +	dev->driver = NULL;
> > +	dev->device.driver = NULL;
> > +
> > +	return 0;
> > +}
> > +
> > +/*
> > + * Call the probe() function of all registered driver for the given device.
> > + * Return < 0 if initialization failed.
> > + * Return 1 if no driver is found for this device.
> > + */
> > +static int
> > +auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
> > +{
> > +	struct rte_auxiliary_driver *drv;
> > +	int rc;
> > +
> > +	if (dev == NULL)
> > +		return -EINVAL;
> > +
> > +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> > +		if (!drv->match(dev->name))
> > +			continue;
> > +
> > +		rc = rte_auxiliary_probe_one_driver(drv, dev);
> > +		if (rc < 0)
> > +			/* negative value is an error */
> > +			return rc;
> > +		if (rc > 0)
> > +			/* positive value means driver doesn't support it */
> > +			continue;
> > +		return 0;
> > +	}
> > +	return 1;
> > +}
> > +
> > +/*
> > + * Scan the content of the auxiliary bus, and call the probe function for
> > + * all registered drivers to try to probe discovered devices.
> > + */
> > +static int
> > +auxiliary_probe(void)
> > +{
> > +	struct rte_auxiliary_device *dev = NULL;
> > +	size_t probed = 0, failed = 0;
> > +	int ret = 0;
> > +
> > +	FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
> > +		probed++;
> > +
> > +		ret = auxiliary_probe_all_drivers(dev);
> > +		if (ret < 0) {
> > +			if (ret != -EEXIST) {
> > +				AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
> > +					      dev->name);
> > +				rte_errno = errno;
> > +				failed++;
> > +			}
> > +			ret = 0;
> > +		}
> > +	}
> > +
> > +	return (probed && probed == failed) ? -1 : 0;
> > +}
> > +
> > +static int
> > +auxiliary_parse(const char *name, void *addr)
> > +{
> > +	struct rte_auxiliary_driver *drv = NULL;
> > +	const char **out = addr;
> > +
> > +	/* Allow empty device name "auxiliary:" to bypass entire bus scan. */
> > +	if (strlen(name) == 0)
> > +		return 0;
> > +
> > +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> > +		if (drv->match(name))
> > +			break;
> > +	}
> > +	if (drv != NULL && addr != NULL)
> > +		*out = name;
> > +	return drv != NULL ? 0 : -1;
> > +}
> > +
> > +/* Register a driver */
> > +void
> > +rte_auxiliary_register(struct rte_auxiliary_driver *driver)
> > +{
> > +	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
> > +	driver->bus = &auxiliary_bus;
> > +}
> > +
> > +/* Unregister a driver */
> > +void
> > +rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
> > +{
> > +	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
> > +	driver->bus = NULL;
> > +}
> > +
> > +/* Add a device to auxiliary bus */
> > +void
> > +auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
> > +{
> > +	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
> > +}
> > +
> > +/* Insert a device into a predefined position in auxiliary bus */
> > +void
> > +auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> > +			struct rte_auxiliary_device *new_aux_dev)
> > +{
> > +	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
> > +}
> > +
> > +/* Remove a device from auxiliary bus */
> > +static void
> > +rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
> > +{
> > +	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
> > +}
> > +
> > +static struct rte_device *
> > +auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> > +		      const void *data)
> > +{
> > +	const struct rte_auxiliary_device *pstart;
> > +	struct rte_auxiliary_device *adev;
> > +
> > +	if (start != NULL) {
> > +		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
> > +		adev = TAILQ_NEXT(pstart, next);
> > +	} else {
> > +		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
> > +	}
> > +	while (adev != NULL) {
> > +		if (cmp(&adev->device, data) == 0)
> > +			return &adev->device;
> > +		adev = TAILQ_NEXT(adev, next);
> > +	}
> > +	return NULL;
> > +}
> > +
> > +static int
> > +auxiliary_plug(struct rte_device *dev)
> > +{
> > +	if (!auxiliary_dev_exists(dev->name))
> > +		return -ENOENT;
> > +	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
> > +}
> > +
> > +static int
> > +auxiliary_unplug(struct rte_device *dev)
> > +{
> > +	struct rte_auxiliary_device *adev;
> > +	int ret;
> > +
> > +	adev = RTE_DEV_TO_AUXILIARY(dev);
> > +	ret = rte_auxiliary_driver_remove_dev(adev);
> > +	if (ret == 0) {
> > +		rte_auxiliary_remove_device(adev);
> > +		rte_devargs_remove(dev->devargs);
> > +		free(adev);
> > +	}
> > +	return ret;
> > +}
> > +
> > +static int
> > +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
> > +{
> > +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> > +
> > +	if (dev == NULL || aux_dev->driver == NULL) {
> > +		rte_errno = EINVAL;
> > +		return -1;
> > +	}
> > +	if (aux_dev->driver->dma_map == NULL) {
> > +		rte_errno = ENOTSUP;
> > +		return -1;
> > +	}
> > +	return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
> > +}
> > +
> > +static int
> > +auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
> > +		    size_t len)
> > +{
> > +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> > +
> > +	if (dev == NULL || aux_dev->driver == NULL) {
> > +		rte_errno = EINVAL;
> > +		return -1;
> > +	}
> > +	if (aux_dev->driver->dma_unmap == NULL) {
> > +		rte_errno = ENOTSUP;
> > +		return -1;
> > +	}
> > +	return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
> > +}
> > +
> > +bool
> > +auxiliary_is_ignored_device(const char *name)
> > +{
> > +	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
> > +
> > +	switch (auxiliary_bus.bus.conf.scan_mode) {
> > +	case RTE_BUS_SCAN_ALLOWLIST:
> > +		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
> > +			return false;
> > +		break;
> > +	case RTE_BUS_SCAN_UNDEFINED:
> > +	case RTE_BUS_SCAN_BLOCKLIST:
> > +		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
> > +			return false;
> > +		break;
> > +	}
> > +	return true;
> > +}
> > +
> > +static enum rte_iova_mode
> > +auxiliary_get_iommu_class(void)
> > +{
> > +	const struct rte_auxiliary_driver *drv;
> > +
> > +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> > +		if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
> > +			return RTE_IOVA_VA;
> > +	}
> > +
> > +	return RTE_IOVA_DC;
> > +}
> > +
> > +struct rte_auxiliary_bus auxiliary_bus = {
> > +	.bus = {
> > +		.scan = auxiliary_scan,
> > +		.probe = auxiliary_probe,
> > +		.find_device = auxiliary_find_device,
> > +		.plug = auxiliary_plug,
> > +		.unplug = auxiliary_unplug,
> > +		.parse = auxiliary_parse,
> > +		.dma_map = auxiliary_dma_map,
> > +		.dma_unmap = auxiliary_dma_unmap,
> > +		.get_iommu_class = auxiliary_get_iommu_class,
> > +		.dev_iterate = auxiliary_dev_iterate,
> > +	},
> > +	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
> > +	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
> > +};
> > +
> > +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> > +RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
> > diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
> > new file mode 100644
> > index 0000000000..cd3fa56cb4
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/auxiliary_params.c
> > @@ -0,0 +1,59 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> > + */
> > +
> > +#include <string.h>
> > +
> > +#include <rte_bus.h>
> > +#include <rte_dev.h>
> > +#include <rte_errno.h>
> > +#include <rte_kvargs.h>
> > +
> > +#include "private.h"
> > +#include "rte_bus_auxiliary.h"
> > +
> > +enum auxiliary_params {
> > +	RTE_AUXILIARY_PARAM_NAME,
> > +};
> > +
> > +static const char * const auxiliary_params_keys[] = {
> > +	[RTE_AUXILIARY_PARAM_NAME] = "name",
> > +};
> > +
> > +static int
> > +auxiliary_dev_match(const struct rte_device *dev,
> > +	      const void *_kvlist)
> > +{
> > +	const struct rte_kvargs *kvlist = _kvlist;
> > +	int ret;
> > +
> > +	ret = rte_kvargs_process(kvlist,
> > +			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
> > +			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
> > +
> > +	return ret != 0 ? -1 : 0;
> > +}
> > +
> > +void *
> > +auxiliary_dev_iterate(const void *start,
> > +		    const char *str,
> > +		    const struct rte_dev_iterator *it __rte_unused)
> > +{
> > +	rte_bus_find_device_t find_device;
> > +	struct rte_kvargs *kvargs = NULL;
> > +	struct rte_device *dev;
> > +
> > +	if (str != NULL) {
> > +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> > +		if (kvargs == NULL) {
> > +			AUXILIARY_LOG(ERR, "cannot parse argument list %s",
> > +				      str);
> > +			rte_errno = EINVAL;
> > +			return NULL;
> > +		}
> > +	}
> > +	find_device = auxiliary_bus.bus.find_device;
> > +	dev = find_device(start, auxiliary_dev_match, kvargs);
> > +	rte_kvargs_free(kvargs);
> > +	return dev;
> > +}
> > diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
> > new file mode 100644
> > index 0000000000..8464487971
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> > @@ -0,0 +1,141 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> > + */
> > +
> > +#include <string.h>
> > +#include <dirent.h>
> > +
> > +#include <rte_log.h>
> > +#include <rte_bus.h>
> > +#include <rte_malloc.h>
> > +#include <rte_devargs.h>
> > +#include <rte_memcpy.h>
> > +#include <eal_filesystem.h>
> > +
> > +#include "../rte_bus_auxiliary.h"
> > +#include "../private.h"
> > +
> > +#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
> > +
> > +/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
> > +static int
> > +auxiliary_scan_one(const char *dirname, const char *name)
> > +{
> > +	struct rte_auxiliary_device *dev;
> > +	struct rte_auxiliary_device *dev2;
> > +	char filename[PATH_MAX];
> > +	unsigned long tmp;
> > +	int ret;
> > +
> > +	dev = malloc(sizeof(*dev));
> > +	if (dev == NULL)
> > +		return -1;
> > +
> > +	memset(dev, 0, sizeof(*dev));
> > +	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
> > +		free(dev);
> > +		return -1;
> > +	}
> > +	dev->device.name = dev->name;
> > +	dev->device.bus = &auxiliary_bus.bus;
> > +
> > +	/* Get NUMA node, default to 0 if not present */
> > +	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
> > +		 dirname, name);
> > +	if (access(filename, F_OK) != -1) {
> > +		if (eal_parse_sysfs_value(filename, &tmp) == 0)
> > +			dev->device.numa_node = tmp;
> > +		else
> > +			dev->device.numa_node = -1;
> > +	} else {
> > +		dev->device.numa_node = 0;
> > +	}
> > +
> > +	auxiliary_on_scan(dev);
> > +
> > +	/* Device is valid, add in list (sorted) */
> > +	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
> > +		ret = strcmp(dev->name, dev2->name);
> > +		if (ret > 0)
> > +			continue;
> > +		if (ret < 0) {
> > +			auxiliary_insert_device(dev2, dev);
> > +		} else { /* already registered */
> > +			if (rte_dev_is_probed(&dev2->device) &&
> > +			    dev2->device.devargs != dev->device.devargs) {
> > +				/* To probe device with new devargs. */
> > +				rte_devargs_remove(dev2->device.devargs);
> > +				auxiliary_on_scan(dev2);
> > +			}
> > +			free(dev);
> > +		}
> > +		return 0;
> > +	}
> > +	auxiliary_add_device(dev);
> > +	return 0;
> > +}
> > +
> > +/*
> > + * Test whether the auxiliary device exist
> 
> Missing full stop above.
> 
> > + */
> > +bool
> > +auxiliary_dev_exists(const char *name)
> > +{
> > +	DIR *dir;
> > +	char dirname[PATH_MAX];
> > +
> > +	snprintf(dirname, sizeof(dirname), "%s/%s",
> > +		 AUXILIARY_SYSFS_PATH, name);
> > +	dir = opendir(dirname);
> > +	if (dir == NULL)
> > +		return false;
> > +	closedir(dir);
> > +	return true;
> > +}
> > +
> > +/*
> > + * Scan the devices in the auxiliary bus
> 
> Missing full stop above.
> 
> > + */
> > +int
> > +auxiliary_scan(void)
> > +{
> > +	struct dirent *e;
> > +	DIR *dir;
> > +	char dirname[PATH_MAX];
> > +	struct rte_auxiliary_driver *drv;
> > +
> > +	dir = opendir(AUXILIARY_SYSFS_PATH);
> > +	if (dir == NULL) {
> > +		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?",
> > +			      AUXILIARY_SYSFS_PATH);
> > +		return 0;
> > +	}
> > +
> > +	while ((e = readdir(dir)) != NULL) {
> > +		if (e->d_name[0] == '.')
> > +			continue;
> > +
> > +		if (auxiliary_is_ignored_device(e->d_name))
> > +			continue;
> > +
> > +		snprintf(dirname, sizeof(dirname), "%s/%s",
> > +			 AUXILIARY_SYSFS_PATH, e->d_name);
> > +
> > +		/* Ignore if no driver can handle. */
> > +		FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> > +			if (drv->match(e->d_name))
> > +				break;
> > +		}
> > +		if (drv == NULL)
> > +			continue;
> > +
> > +		if (auxiliary_scan_one(dirname, e->d_name) < 0)
> > +			goto error;
> > +	}
> > +	closedir(dir);
> > +	return 0;
> > +
> > +error:
> > +	closedir(dir);
> > +	return -1;
> > +}
> > diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
> > new file mode 100644
> > index 0000000000..357550eff7
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/meson.build
> > @@ -0,0 +1,16 @@
> > +# SPDX-License-Identifier: BSD-3-Clause
> > +# Copyright (c) 2021 NVIDIA Corporation & Affiliates
> > +
> > +headers = files(
> > +        'rte_bus_auxiliary.h',
> > +)
> > +sources = files(
> > +        'auxiliary_common.c',
> > +        'auxiliary_params.c',
> > +)
> > +if is_linux
> > +    sources += files(
> > +        'linux/auxiliary.c',
> > +    )
> > +endif
> > +deps += ['kvargs']
> > diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
> > new file mode 100644
> > index 0000000000..cb3e849993
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/private.h
> > @@ -0,0 +1,74 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> > + */
> > +
> > +#ifndef AUXILIARY_PRIVATE_H
> 
> May be add BUS_ prefix at leaat?
> 
> > +#define AUXILIARY_PRIVATE_H
> > +
> > +#include <stdbool.h>
> > +#include <stdio.h>
> > +
> > +#include "rte_bus_auxiliary.h"
> > +
> > +extern struct rte_auxiliary_bus auxiliary_bus;
> > +extern int auxiliary_bus_logtype;
> > +
> > +#define AUXILIARY_LOG(level, ...) \
> > +	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, \
> > +		RTE_FMT("auxiliary bus: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
> > +			RTE_FMT_TAIL(__VA_ARGS__,)))
> > +
> > +/* Auxiliary bus iterators */
> > +#define FOREACH_DEVICE_ON_AUXILIARY_BUS(p) \
> > +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> > +
> > +#define FOREACH_DRIVER_ON_AUXILIARY_BUS(p) \
> > +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> > +
> > +bool auxiliary_dev_exists(const char *name);
> > +
> > +/*
> > + * Scan the content of the auxiliary bus, and the devices in the devices
> > + * list.
> > + */
> > +int auxiliary_scan(void);
> > +
> > +/*
> > + * Update a device being scanned.
> > + */
> > +void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
> > +
> > +/*
> > + * Validate whether a device with given auxiliary device should be ignored
> > + * or not.
> > + */
> > +bool auxiliary_is_ignored_device(const char *name);
> > +
> > +/*
> > + * Add an auxiliary device to the auxiliary bus (append to auxiliary device
> > + * list). This function also updates the bus references of the auxiliary
> > + * device and the generic device object embedded within.
> > + */
> > +void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
> > +
> > +/*
> > + * Insert an auxiliary device in the auxiliary bus at a particular location
> > + * in the device list. It also updates the auxiliary bus reference of the
> > + * new devices to be inserted.
> > + */
> > +void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> > +			     struct rte_auxiliary_device *new_aux_dev);
> > +
> > +/*
> > + * Match the auxiliary driver and device by driver function
> 
> Missing full stop.
> 
> > + */
> > +bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> > +		     const struct rte_auxiliary_device *aux_dev);
> > +
> > +/*
> > + * Iterate over devices, matching any device against the provided string
> 
> Missing full stop.
> 
> > + */
> > +void *auxiliary_dev_iterate(const void *start, const char *str,
> > +			    const struct rte_dev_iterator *it);
> > +
> > +#endif /* AUXILIARY_PRIVATE_H */
> > diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > new file mode 100644
> > index 0000000000..16b147e387
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> > @@ -0,0 +1,201 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> > + */
> > +
> > +#ifndef RTE_BUS_AUXILIARY_H
> > +#define RTE_BUS_AUXILIARY_H
> > +
> > +/**
> > + * @file
> > + *
> > + * Auxiliary Bus Interface.
> > + */
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <limits.h>
> > +#include <errno.h>
> > +#include <sys/queue.h>
> > +#include <stdint.h>
> > +#include <inttypes.h>
> > +
> > +#include <rte_debug.h>
> > +#include <rte_interrupts.h>
> > +#include <rte_dev.h>
> > +#include <rte_bus.h>
> > +#include <rte_kvargs.h>
> > +
> > +#define RTE_BUS_AUXILIARY_NAME "auxiliary"
> > +
> > +/* Forward declarations */
> > +struct rte_auxiliary_driver;
> > +struct rte_auxiliary_bus;
> > +struct rte_auxiliary_device;
> > +
> > +/**
> > + * Match function for the driver to decide if device can be handled.
> > + *
> > + * @param name
> > + *   Pointer to the auxiliary device name.
> > + * @return
> > + *   Whether the driver can handle the auxiliary device.
> > + */
> > +typedef bool(rte_auxiliary_match_t)(const char *name);
> > +
> > +/**
> > + * Initialization function for the driver called during auxiliary probing.
> > + *
> > + * @param drv
> > + *   Pointer to the auxiliary driver.
> > + * @param dev
> > + *   Pointer to the auxiliary device.
> > + * @return
> > + *   - 0 On success.
> > + *   - Negative value and rte_errno is set otherwise.
> > + */
> > +typedef int(rte_auxiliary_probe_t)(struct rte_auxiliary_driver *drv,
> > +				    struct rte_auxiliary_device *dev);
> > +
> > +/**
> > + * Uninitialization function for the driver called during hotplugging.
> > + *
> > + * @param dev
> > + *   Pointer to the auxiliary device.
> > + * @return
> > + *   - 0 On success.
> > + *   - Negative value and rte_errno is set otherwise.
> > + */
> > +typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
> > +
> > +/**
> > + * Driver-specific DMA mapping. After a successful call the device
> > + * will be able to read/write from/to this segment.
> > + *
> > + * @param dev
> > + *   Pointer to the auxiliary device.
> > + * @param addr
> > + *   Starting virtual address of memory to be mapped.
> > + * @param iova
> > + *   Starting IOVA address of memory to be mapped.
> > + * @param len
> > + *   Length of memory segment being mapped.
> > + * @return
> > + *   - 0 On success.
> > + *   - Negative value and rte_errno is set otherwise.
> > + */
> > +typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
> > +				       void *addr, uint64_t iova, size_t len);
> > +
> > +/**
> > + * Driver-specific DMA un-mapping. After a successful call the device
> > + * will not be able to read/write from/to this segment.
> > + *
> > + * @param dev
> > + *   Pointer to the auxiliary device.
> > + * @param addr
> > + *   Starting virtual address of memory to be unmapped.
> > + * @param iova
> > + *   Starting IOVA address of memory to be unmapped.
> > + * @param len
> > + *   Length of memory segment being unmapped.
> > + * @return
> > + *   - 0 On success.
> > + *   - Negative value and rte_errno is set otherwise.
> > + */
> > +typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> > +					 void *addr, uint64_t iova, size_t len);
> > +
> > +/**
> > + * A structure describing an auxiliary device.
> > + */
> > +struct rte_auxiliary_device {
> > +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> > +	struct rte_device device;                 /**< Inherit core device */
> > +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> > +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> > +	struct rte_auxiliary_driver *driver;      /**< Device driver */
> > +};
> > +
> > +/** List of auxiliary devices */
> > +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> > +/** List of auxiliary drivers */
> > +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> 
> Shouldn't we hide rte_auxiliary_device inside the library take
> API/ABI stability into account? Or will be it DPDK internal anyway? If
> so, it should be done INTERNAL from the very
> beginning.
> 
> > +
> > +/**
> > + * Structure describing the auxiliary bus
> > + */
> > +struct rte_auxiliary_bus {
> > +	struct rte_bus bus;                  /**< Inherit the generic class */
> > +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> > +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
> > +};
> 
> It looks internal. The following forward declaration should be
> sufficient to build.
> 
> struct rte_auxiliary_bus;
> 
> 
> > +
> > +/**
> > + * A structure describing an auxiliary driver.
> > + */
> > +struct rte_auxiliary_driver {
> > +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> > +	struct rte_driver driver;             /**< Inherit core driver. */
> > +	struct rte_auxiliary_bus *bus;        /**< Auxiliary bus reference. */
> > +	rte_auxiliary_match_t *match;         /**< Device match function. */
> > +	rte_auxiliary_probe_t *probe;         /**< Device probe function. */
> > +	rte_auxiliary_remove_t *remove;       /**< Device remove function. */
> > +	rte_auxiliary_dma_map_t *dma_map;     /**< Device DMA map function. */
> > +	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device DMA unmap function. */
> > +	uint32_t drv_flags;                   /**< Flags RTE_AUXILIARY_DRV_*. */
> > +};
> > +
> > +/**
> > + * @internal
> > + * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
> > + */
> > +#define RTE_DEV_TO_AUXILIARY(ptr) \
> > +	container_of(ptr, struct rte_auxiliary_device, device)
> > +
> > +#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
> > +	container_of(ptr, const struct rte_auxiliary_device, device)
> > +
> > +#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
> > +	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
> > +
> > +/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
> > +#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
> > +
> > +/**
> 
> Don't we need EXPERIMENTAL notice here?
> 
> > + * Register an auxiliary driver.
> > + *
> > + * @param driver
> > + *   A pointer to a rte_auxiliary_driver structure describing the driver
> > + *   to be registered.
> > + */
> > +__rte_experimental
> > +void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
> > +
> > +/** Helper for auxiliary device registration from driver instance */
> > +#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv) \
> > +	RTE_INIT(auxiliaryinitfn_ ##nm) \
> > +	{ \
> > +		(auxiliary_drv).driver.name = RTE_STR(nm); \
> > +		rte_auxiliary_register(&(auxiliary_drv)); \
> > +	} \
> > +	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
> > +
> > +/**
> 
> Don't we need EXPERIMENTAL notice here?
> 
> > + * Unregister an auxiliary driver.
> > + *
> > + * @param driver
> > + *   A pointer to a rte_auxiliary_driver structure describing the driver
> > + *   to be unregistered.
> > + */
> > +__rte_experimental
> > +void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
> > +
> > +#ifdef __cplusplus
> > +}
> > +#endif
> > +
> > +#endif /* RTE_BUS_AUXILIARY_H */
> > diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
> > new file mode 100644
> > index 0000000000..a52260657c
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/version.map
> > @@ -0,0 +1,7 @@
> > +EXPERIMENTAL {
> > +	global:
> > +
> > +	# added in 21.08
> > +	rte_auxiliary_register;
> > +	rte_auxiliary_unregister;
> > +};
> > diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
> > index 410058de3a..45eab5233d 100644
> > --- a/drivers/bus/meson.build
> > +++ b/drivers/bus/meson.build
> > @@ -2,6 +2,7 @@
> >  # Copyright(c) 2017 Intel Corporation
> >
> >  drivers = [
> > +        'auxiliary',
> >          'dpaa',
> >          'fslmc',
> >          'ifpga',
> >

[dpdk-dev] [PATCH v8 1/2] devargs: add common key definition

[Xueming Li]

Adds common devargs key definition for "bus", "class" and "driver".

Acked-by: Thomas Monjalon <thomas@monjalon.net>
Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |  2 --
 drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
 drivers/common/sfc_efx/sfc_efx.c        |  7 +++----
 drivers/common/sfc_efx/sfc_efx.h        |  2 --
 drivers/net/bonding/rte_eth_bond_args.c |  2 +-
 drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
 drivers/net/iavf/iavf_ethdev.c          |  5 ++---
 drivers/net/mlx5/mlx5.c                 |  4 ++--
 drivers/net/sfc/sfc_kvargs.c            |  2 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
 lib/eal/common/eal_common_devargs.c     | 12 ++++++------
 lib/eal/include/rte_devargs.h           | 24 ++++++++++++++++++++++++
 12 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..306f2f1ab7 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -208,8 +208,6 @@ __rte_internal
 int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
 
 
-#define MLX5_CLASS_ARG_NAME "class"
-
 enum mlx5_class {
 	MLX5_CLASS_INVALID,
 	MLX5_CLASS_NET = RTE_BIT64(0),
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..34747c4e07 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,
 static int
 parse_class_options(const struct rte_devargs *devargs)
 {
-	const char *key = MLX5_CLASS_ARG_NAME;
+	const char *key = RTE_DEVARGS_KEY_CLASS;
 	struct rte_kvargs *kvlist;
 	int ret = 0;
 
diff --git a/drivers/common/sfc_efx/sfc_efx.c b/drivers/common/sfc_efx/sfc_efx.c
index 0b78933d9f..2dc5545760 100644
--- a/drivers/common/sfc_efx/sfc_efx.c
+++ b/drivers/common/sfc_efx/sfc_efx.c
@@ -42,7 +42,6 @@ enum sfc_efx_dev_class
 sfc_efx_dev_class_get(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvargs;
-	const char *key = SFC_EFX_KVARG_DEV_CLASS;
 	enum sfc_efx_dev_class dev_class = SFC_EFX_DEV_CLASS_NET;
 
 	if (devargs == NULL)
@@ -52,9 +51,9 @@ sfc_efx_dev_class_get(struct rte_devargs *devargs)
 	if (kvargs == NULL)
 		return dev_class;
 
-	if (rte_kvargs_count(kvargs, key) != 0) {
-		rte_kvargs_process(kvargs, key, sfc_efx_kvarg_dev_class_handler,
-				   &dev_class);
+	if (rte_kvargs_count(kvargs, RTE_DEVARGS_KEY_CLASS) != 0) {
+		rte_kvargs_process(kvargs, RTE_DEVARGS_KEY_CLASS,
+				   sfc_efx_kvarg_dev_class_handler, &dev_class);
 	}
 
 	rte_kvargs_free(kvargs);
diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
index 6b6164cb1f..c16eca60f3 100644
--- a/drivers/common/sfc_efx/sfc_efx.h
+++ b/drivers/common/sfc_efx/sfc_efx.h
@@ -19,8 +19,6 @@
 extern "C" {
 #endif
 
-#define SFC_EFX_KVARG_DEV_CLASS	"class"
-
 enum sfc_efx_dev_class {
 	SFC_EFX_DEV_CLASS_INVALID = 0,
 	SFC_EFX_DEV_CLASS_NET,
diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
index 764b1b8c8e..5406e1c934 100644
--- a/drivers/net/bonding/rte_eth_bond_args.c
+++ b/drivers/net/bonding/rte_eth_bond_args.c
@@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
 	PMD_BOND_SOCKET_ID_KVARG,
 	PMD_BOND_MAC_ADDR_KVARG,
 	PMD_BOND_AGG_MODE_KVARG,
-	"driver",
+	RTE_DEVARGS_KEY_DRIVER,
 	NULL
 };
 
diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
index 385ebedcd3..0cfe13b7b2 100644
--- a/drivers/net/i40e/i40e_ethdev_vf.c
+++ b/drivers/net/i40e/i40e_ethdev_vf.c
@@ -1660,7 +1660,6 @@ static int
 i40evf_driver_selected(struct rte_devargs *devargs)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (devargs == NULL)
@@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       i40evf_check_driver_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 5290588b17..472538181e 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -2448,7 +2448,6 @@ static int
 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 {
 	struct rte_kvargs *kvlist;
-	const char *key = "driver";
 	int ret = 0;
 
 	if (device_id != IAVF_DEV_ID_VF &&
@@ -2464,13 +2463,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
 	if (kvlist == NULL)
 		return 0;
 
-	if (!rte_kvargs_count(kvlist, key))
+	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
 		goto exit;
 
 	/* i40evf driver selected when there's a key-value pair:
 	 * driver=i40evf
 	 */
-	if (rte_kvargs_process(kvlist, key,
+	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
 			       iavf_drv_i40evf_check_handler, NULL) < 0)
 		goto exit;
 
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..d0faa45944 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->max_dump_files_num = tmp;
 	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
 		config->lro.timeout = tmp;
-	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
+	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
 		DRV_LOG(DEBUG, "class argument is %s.", val);
 	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
 		config->log_hp_size = tmp;
@@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_REPRESENTOR,
 		MLX5_MAX_DUMP_FILES_NUM,
 		MLX5_LRO_TIMEOUT_USEC,
-		MLX5_CLASS_ARG_NAME,
+		RTE_DEVARGS_KEY_CLASS,
 		MLX5_HP_BUF_SIZE,
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c
index 0efa92ed28..974c05e68e 100644
--- a/drivers/net/sfc/sfc_kvargs.c
+++ b/drivers/net/sfc/sfc_kvargs.c
@@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
 		SFC_KVARG_TX_DATAPATH,
 		SFC_KVARG_FW_VARIANT,
 		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
-		SFC_EFX_KVARG_DEV_CLASS,
+		RTE_DEVARGS_KEY_CLASS,
 		NULL,
 	};
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index e5e03e6582..8b5bfd8c3d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
 	unsigned long tmp;
 	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
 
-	if (strcmp(key, "class") == 0)
+	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
 		return 0;
 	errno = 0;
 	tmp = strtoul(val, NULL, 0);
diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
index b31ac879a9..23aaf8b7e4 100644
--- a/lib/eal/common/eal_common_devargs.c
+++ b/lib/eal/common/eal_common_devargs.c
@@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		const char *str;
 		struct rte_kvargs *kvlist;
 	} layers[] = {
-		{ "bus=",    NULL, NULL, },
-		{ "class=",  NULL, NULL, },
-		{ "driver=", NULL, NULL, },
+		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
+		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
+		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
 	};
 	struct rte_kvargs_pair *kv = NULL;
 	struct rte_class *cls = NULL;
@@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 		if (layers[i].kvlist == NULL)
 			continue;
 		kv = &layers[i].kvlist->pairs[0];
-		if (strcmp(kv->key, "bus") == 0) {
+		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
 			bus = rte_bus_find_by_name(kv->value);
 			if (bus == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n",
@@ -126,7 +126,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "class") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
 			cls = rte_class_find_by_name(kv->value);
 			if (cls == NULL) {
 				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n",
@@ -134,7 +134,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
 				ret = -EFAULT;
 				goto get_out;
 			}
-		} else if (strcmp(kv->key, "driver") == 0) {
+		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
 			/* Ignore */
 			continue;
 		}
diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h
index 1e595b3c51..cd90944fe8 100644
--- a/lib/eal/include/rte_devargs.h
+++ b/lib/eal/include/rte_devargs.h
@@ -25,6 +25,30 @@ extern "C" {
 #include <rte_compat.h>
 #include <rte_bus.h>
 
+/**
+ * Bus type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't use this key as bus type
+ * is resolved as first optional value separated by ":".
+ */
+#define RTE_DEVARGS_KEY_BUS "bus"
+
+/**
+ * Class type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't parse class type. PMD driver is
+ * encouraged to use this key to resolve class type.
+ */
+#define RTE_DEVARGS_KEY_CLASS "class"
+
+/**
+ * Driver type key in global devargs syntax.
+ *
+ * Legacy devargs parser doesn't parse driver type. PMD driver is
+ * encouraged to use this key to resolve driver type.
+ */
+#define RTE_DEVARGS_KEY_DRIVER "driver"
+
 /**
  * Type of generic device
  */
-- 
2.25.1

[dpdk-dev] [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming Li]

Auxiliary bus [1] provides a way to split function into child-devices
representing sub-domains of functionality. Each auxiliary device
represents a part of its parent functionality.

Auxiliary device is identified by unique device name, sysfs path:
  /sys/bus/auxiliary/devices/<name>

Devargs legacy syntax of auxiliary device:
  -a auxiliary:<name>[,args...]
Devargs generic syntax of auxiliary device:
  -a bus=auxiliary,name=<name>/class=<class>/driver=<driver>[,args...]

[1] kernel auxiliary bus document:
https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Cc: Wang Haiyue <haiyue.wang@intel.com>
Cc: Thomas Monjalon <thomas@monjalon.net>
Cc: Kinsella Ray <mdr@ashroe.eu>
Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
---
 MAINTAINERS                               |   5 +
 doc/guides/rel_notes/release_21_08.rst    |   6 +
 drivers/bus/auxiliary/auxiliary_common.c  | 411 ++++++++++++++++++++++
 drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
 drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
 drivers/bus/auxiliary/meson.build         |  16 +
 drivers/bus/auxiliary/private.h           |  91 +++++
 drivers/bus/auxiliary/rte_bus_auxiliary.h | 193 ++++++++++
 drivers/bus/auxiliary/version.map         |   7 +
 drivers/bus/meson.build                   |   1 +
 10 files changed, 930 insertions(+)
 create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
 create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
 create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
 create mode 100644 drivers/bus/auxiliary/meson.build
 create mode 100644 drivers/bus/auxiliary/private.h
 create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
 create mode 100644 drivers/bus/auxiliary/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..9b0d0d4348 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst
 Bus Drivers
 -----------
 
+Auxiliary bus driver - EXPERIMENTAL
+M: Parav Pandit <parav@nvidia.com>
+M: Xueming Li <xuemingl@nvidia.com>
+F: drivers/bus/auxiliary/
+
 Intel FPGA bus
 M: Rosen Xu <rosen.xu@intel.com>
 F: drivers/bus/ifpga/
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..e7ef4c8a05 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added auxiliary bus support.**
+
+  Auxiliary bus provides a way to split function into child-devices
+  representing sub-domains of functionality. Each auxiliary device
+  represents a part of its parent functionality.
+
 
 Removed Items
 -------------
diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
new file mode 100644
index 0000000000..abe6aef098
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_common.c
@@ -0,0 +1,411 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/queue.h>
+#include <rte_errno.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_per_lcore.h>
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_eal_paging.h>
+#include <rte_string_fns.h>
+#include <rte_common.h>
+#include <rte_devargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+static struct rte_devargs *
+auxiliary_devargs_lookup(const char *name)
+{
+	struct rte_devargs *devargs;
+
+	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
+		if (strcmp(devargs->name, name) == 0)
+			return devargs;
+	}
+	return NULL;
+}
+
+/*
+ * Test whether the auxiliary device exist.
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak bool
+auxiliary_dev_exists(const char *name)
+{
+	RTE_SET_USED(name);
+	return false;
+}
+
+/*
+ * Scan the devices in the auxiliary bus.
+ *
+ * Stub for OS not supporting auxiliary bus.
+ */
+__rte_weak int
+auxiliary_scan(void)
+{
+	return 0;
+}
+
+/*
+ * Update a device's devargs being scanned.
+ *
+ * @param aux_dev
+ *	AUXILIARY device.
+ */
+void
+auxiliary_on_scan(struct rte_auxiliary_device *aux_dev)
+{
+	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
+}
+
+/*
+ * Match the auxiliary driver and device using driver function.
+ */
+bool
+auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		const struct rte_auxiliary_device *aux_dev)
+{
+	if (aux_drv->match == NULL)
+		return false;
+	return aux_drv->match(aux_dev->name);
+}
+
+/*
+ * Call the probe() function of the driver.
+ */
+static int
+rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
+			       struct rte_auxiliary_device *dev)
+{
+	enum rte_iova_mode iova_mode;
+	int ret;
+
+	if (drv == NULL || dev == NULL)
+		return -EINVAL;
+
+	/* Check if driver supports it. */
+	if (!auxiliary_match(drv, dev))
+		/* Match of device and driver failed */
+		return 1;
+
+	/* No initialization when marked as blocked, return without error. */
+	if (dev->device.devargs != NULL &&
+	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+		AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
+		return -1;
+	}
+
+	if (dev->device.numa_node < 0) {
+		AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting NUMA node to 0");
+		dev->device.numa_node = 0;
+	}
+
+	iova_mode = rte_eal_iova_mode();
+	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
+	    iova_mode != RTE_IOVA_VA) {
+		AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
+			      drv->driver.name);
+		return -EINVAL;
+	}
+
+	dev->driver = drv;
+
+	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (NUMA node %i)",
+		      drv->driver.name, dev->name, dev->device.numa_node);
+	ret = drv->probe(drv, dev);
+	if (ret != 0)
+		dev->driver = NULL;
+	else
+		dev->device.driver = &drv->driver;
+
+	return ret;
+}
+
+/*
+ * Call the remove() function of the driver.
+ */
+static int
+rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *drv;
+	int ret = 0;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	drv = dev->driver;
+
+	AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA node %i",
+		      drv->driver.name, dev->name, dev->device.numa_node);
+
+	if (drv->remove != NULL) {
+		ret = drv->remove(dev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* clear driver structure */
+	dev->driver = NULL;
+	dev->device.driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the probe() function of all registered driver for the given device.
+ * Return < 0 if initialization failed.
+ * Return 1 if no driver is found for this device.
+ */
+static int
+auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev)
+{
+	struct rte_auxiliary_driver *drv;
+	int rc;
+
+	if (dev == NULL)
+		return -EINVAL;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if (!drv->match(dev->name))
+			continue;
+
+		rc = rte_auxiliary_probe_one_driver(drv, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return rc;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Scan the content of the auxiliary bus, and call the probe function for
+ * all registered drivers to try to probe discovered devices.
+ */
+static int
+auxiliary_probe(void)
+{
+	struct rte_auxiliary_device *dev = NULL;
+	size_t probed = 0, failed = 0;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
+		probed++;
+
+		ret = auxiliary_probe_all_drivers(dev);
+		if (ret < 0) {
+			if (ret != -EEXIST) {
+				AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
+					      dev->name);
+				rte_errno = errno;
+				failed++;
+			}
+			ret = 0;
+		}
+	}
+
+	return (probed && probed == failed) ? -1 : 0;
+}
+
+static int
+auxiliary_parse(const char *name, void *addr)
+{
+	struct rte_auxiliary_driver *drv = NULL;
+	const char **out = addr;
+
+	/* Allow empty device name "auxiliary:" to bypass entire bus scan. */
+	if (strlen(name) == 0)
+		return 0;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if (drv->match(name))
+			break;
+	}
+	if (drv != NULL && addr != NULL)
+		*out = name;
+	return drv != NULL ? 0 : -1;
+}
+
+/* Register a driver */
+void
+rte_auxiliary_register(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = &auxiliary_bus;
+}
+
+/* Unregister a driver */
+void
+rte_auxiliary_unregister(struct rte_auxiliary_driver *driver)
+{
+	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
+	driver->bus = NULL;
+}
+
+/* Add a device to auxiliary bus */
+void
+auxiliary_add_device(struct rte_auxiliary_device *aux_dev)
+{
+	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next);
+}
+
+/* Insert a device into a predefined position in auxiliary bus */
+void
+auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			struct rte_auxiliary_device *new_aux_dev)
+{
+	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next);
+}
+
+/* Remove a device from auxiliary bus */
+static void
+rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
+{
+	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next);
+}
+
+static struct rte_device *
+auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
+		      const void *data)
+{
+	const struct rte_auxiliary_device *pstart;
+	struct rte_auxiliary_device *adev;
+
+	if (start != NULL) {
+		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
+		adev = TAILQ_NEXT(pstart, next);
+	} else {
+		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
+	}
+	while (adev != NULL) {
+		if (cmp(&adev->device, data) == 0)
+			return &adev->device;
+		adev = TAILQ_NEXT(adev, next);
+	}
+	return NULL;
+}
+
+static int
+auxiliary_plug(struct rte_device *dev)
+{
+	if (!auxiliary_dev_exists(dev->name))
+		return -ENOENT;
+	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
+}
+
+static int
+auxiliary_unplug(struct rte_device *dev)
+{
+	struct rte_auxiliary_device *adev;
+	int ret;
+
+	adev = RTE_DEV_TO_AUXILIARY(dev);
+	ret = rte_auxiliary_driver_remove_dev(adev);
+	if (ret == 0) {
+		rte_auxiliary_remove_device(adev);
+		rte_devargs_remove(dev->devargs);
+		free(adev);
+	}
+	return ret;
+}
+
+static int
+auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || aux_dev->driver == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_map == NULL) {
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return aux_dev->driver->dma_map(aux_dev, addr, iova, len);
+}
+
+static int
+auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+		    size_t len)
+{
+	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
+
+	if (dev == NULL || aux_dev->driver == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	if (aux_dev->driver->dma_unmap == NULL) {
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len);
+}
+
+bool
+auxiliary_is_ignored_device(const char *name)
+{
+	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
+
+	switch (auxiliary_bus.bus.conf.scan_mode) {
+	case RTE_BUS_SCAN_ALLOWLIST:
+		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
+			return false;
+		break;
+	case RTE_BUS_SCAN_UNDEFINED:
+	case RTE_BUS_SCAN_BLOCKLIST:
+		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
+			return false;
+		break;
+	}
+	return true;
+}
+
+static enum rte_iova_mode
+auxiliary_get_iommu_class(void)
+{
+	const struct rte_auxiliary_driver *drv;
+
+	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+		if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
+			return RTE_IOVA_VA;
+	}
+
+	return RTE_IOVA_DC;
+}
+
+struct rte_auxiliary_bus auxiliary_bus = {
+	.bus = {
+		.scan = auxiliary_scan,
+		.probe = auxiliary_probe,
+		.find_device = auxiliary_find_device,
+		.plug = auxiliary_plug,
+		.unplug = auxiliary_unplug,
+		.parse = auxiliary_parse,
+		.dma_map = auxiliary_dma_map,
+		.dma_unmap = auxiliary_dma_unmap,
+		.get_iommu_class = auxiliary_get_iommu_class,
+		.dev_iterate = auxiliary_dev_iterate,
+	},
+	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
+	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
+};
+
+RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
+RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
new file mode 100644
index 0000000000..cd3fa56cb4
--- /dev/null
+++ b/drivers/bus/auxiliary/auxiliary_params.c
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+
+#include <rte_bus.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_kvargs.h>
+
+#include "private.h"
+#include "rte_bus_auxiliary.h"
+
+enum auxiliary_params {
+	RTE_AUXILIARY_PARAM_NAME,
+};
+
+static const char * const auxiliary_params_keys[] = {
+	[RTE_AUXILIARY_PARAM_NAME] = "name",
+};
+
+static int
+auxiliary_dev_match(const struct rte_device *dev,
+	      const void *_kvlist)
+{
+	const struct rte_kvargs *kvlist = _kvlist;
+	int ret;
+
+	ret = rte_kvargs_process(kvlist,
+			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
+			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
+
+	return ret != 0 ? -1 : 0;
+}
+
+void *
+auxiliary_dev_iterate(const void *start,
+		    const char *str,
+		    const struct rte_dev_iterator *it __rte_unused)
+{
+	rte_bus_find_device_t find_device;
+	struct rte_kvargs *kvargs = NULL;
+	struct rte_device *dev;
+
+	if (str != NULL) {
+		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
+		if (kvargs == NULL) {
+			AUXILIARY_LOG(ERR, "cannot parse argument list %s",
+				      str);
+			rte_errno = EINVAL;
+			return NULL;
+		}
+	}
+	find_device = auxiliary_bus.bus.find_device;
+	dev = find_device(start, auxiliary_dev_match, kvargs);
+	rte_kvargs_free(kvargs);
+	return dev;
+}
diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
new file mode 100644
index 0000000000..9bd4ee3295
--- /dev/null
+++ b/drivers/bus/auxiliary/linux/auxiliary.c
@@ -0,0 +1,141 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_log.h>
+#include <rte_bus.h>
+#include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_memcpy.h>
+#include <eal_filesystem.h>
+
+#include "../rte_bus_auxiliary.h"
+#include "../private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+
+/* Scan one auxiliary sysfs entry, and fill the devices list from it. */
+static int
+auxiliary_scan_one(const char *dirname, const char *name)
+{
+	struct rte_auxiliary_device *dev;
+	struct rte_auxiliary_device *dev2;
+	char filename[PATH_MAX];
+	unsigned long tmp;
+	int ret;
+
+	dev = malloc(sizeof(*dev));
+	if (dev == NULL)
+		return -1;
+
+	memset(dev, 0, sizeof(*dev));
+	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
+		free(dev);
+		return -1;
+	}
+	dev->device.name = dev->name;
+	dev->device.bus = &auxiliary_bus.bus;
+
+	/* Get NUMA node, default to 0 if not present */
+	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
+		 dirname, name);
+	if (access(filename, F_OK) != -1) {
+		if (eal_parse_sysfs_value(filename, &tmp) == 0)
+			dev->device.numa_node = tmp;
+		else
+			dev->device.numa_node = -1;
+	} else {
+		dev->device.numa_node = 0;
+	}
+
+	auxiliary_on_scan(dev);
+
+	/* Device is valid, add in list (sorted) */
+	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
+		ret = strcmp(dev->name, dev2->name);
+		if (ret > 0)
+			continue;
+		if (ret < 0) {
+			auxiliary_insert_device(dev2, dev);
+		} else { /* already registered */
+			if (rte_dev_is_probed(&dev2->device) &&
+			    dev2->device.devargs != dev->device.devargs) {
+				/* To probe device with new devargs. */
+				rte_devargs_remove(dev2->device.devargs);
+				auxiliary_on_scan(dev2);
+			}
+			free(dev);
+		}
+		return 0;
+	}
+	auxiliary_add_device(dev);
+	return 0;
+}
+
+/*
+ * Test whether the auxiliary device exist.
+ */
+bool
+auxiliary_dev_exists(const char *name)
+{
+	DIR *dir;
+	char dirname[PATH_MAX];
+
+	snprintf(dirname, sizeof(dirname), "%s/%s",
+		 AUXILIARY_SYSFS_PATH, name);
+	dir = opendir(dirname);
+	if (dir == NULL)
+		return false;
+	closedir(dir);
+	return true;
+}
+
+/*
+ * Scan the devices in the auxiliary bus.
+ */
+int
+auxiliary_scan(void)
+{
+	struct dirent *e;
+	DIR *dir;
+	char dirname[PATH_MAX];
+	struct rte_auxiliary_driver *drv;
+
+	dir = opendir(AUXILIARY_SYSFS_PATH);
+	if (dir == NULL) {
+		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?",
+			      AUXILIARY_SYSFS_PATH);
+		return 0;
+	}
+
+	while ((e = readdir(dir)) != NULL) {
+		if (e->d_name[0] == '.')
+			continue;
+
+		if (auxiliary_is_ignored_device(e->d_name))
+			continue;
+
+		snprintf(dirname, sizeof(dirname), "%s/%s",
+			 AUXILIARY_SYSFS_PATH, e->d_name);
+
+		/* Ignore if no driver can handle. */
+		FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
+			if (drv->match(e->d_name))
+				break;
+		}
+		if (drv == NULL)
+			continue;
+
+		if (auxiliary_scan_one(dirname, e->d_name) < 0)
+			goto error;
+	}
+	closedir(dir);
+	return 0;
+
+error:
+	closedir(dir);
+	return -1;
+}
diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
new file mode 100644
index 0000000000..357550eff7
--- /dev/null
+++ b/drivers/bus/auxiliary/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+headers = files(
+        'rte_bus_auxiliary.h',
+)
+sources = files(
+        'auxiliary_common.c',
+        'auxiliary_params.c',
+)
+if is_linux
+    sources += files(
+        'linux/auxiliary.c',
+    )
+endif
+deps += ['kvargs']
diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h
new file mode 100644
index 0000000000..9987e8b501
--- /dev/null
+++ b/drivers/bus/auxiliary/private.h
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef BUS_AUXILIARY_PRIVATE_H
+#define BUS_AUXILIARY_PRIVATE_H
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "rte_bus_auxiliary.h"
+
+extern struct rte_auxiliary_bus auxiliary_bus;
+extern int auxiliary_bus_logtype;
+
+#define AUXILIARY_LOG(level, ...) \
+	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, \
+		RTE_FMT("auxiliary bus: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
+			RTE_FMT_TAIL(__VA_ARGS__,)))
+
+/* Auxiliary bus iterators */
+#define FOREACH_DEVICE_ON_AUXILIARY_BUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
+
+#define FOREACH_DRIVER_ON_AUXILIARY_BUS(p) \
+		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
+
+/* List of auxiliary devices. */
+TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
+/* List of auxiliary drivers. */
+TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
+
+/*
+ * Structure describing the auxiliary bus
+ */
+struct rte_auxiliary_bus {
+	struct rte_bus bus;                  /* Inherit the generic class */
+	struct rte_auxiliary_device_list device_list;  /* List of devices */
+	struct rte_auxiliary_driver_list driver_list;  /* List of drivers */
+};
+
+/*
+ * Test whether the auxiliary device exist.
+ */
+bool auxiliary_dev_exists(const char *name);
+
+/*
+ * Scan the content of the auxiliary bus, and the devices in the devices
+ * list.
+ */
+int auxiliary_scan(void);
+
+/*
+ * Update a device being scanned.
+ */
+void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Validate whether a device with given auxiliary device should be ignored
+ * or not.
+ */
+bool auxiliary_is_ignored_device(const char *name);
+
+/*
+ * Add an auxiliary device to the auxiliary bus (append to auxiliary device
+ * list). This function also updates the bus references of the auxiliary
+ * device and the generic device object embedded within.
+ */
+void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Insert an auxiliary device in the auxiliary bus at a particular location
+ * in the device list. It also updates the auxiliary bus reference of the
+ * new devices to be inserted.
+ */
+void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
+			     struct rte_auxiliary_device *new_aux_dev);
+
+/*
+ * Match the auxiliary driver and device by driver function.
+ */
+bool auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
+		     const struct rte_auxiliary_device *aux_dev);
+
+/*
+ * Iterate over devices, matching any device against the provided string.
+ */
+void *auxiliary_dev_iterate(const void *start, const char *str,
+			    const struct rte_dev_iterator *it);
+
+#endif /* BUS_AUXILIARY_PRIVATE_H */
diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
new file mode 100644
index 0000000000..b7a8a03be4
--- /dev/null
+++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_BUS_AUXILIARY_H
+#define RTE_BUS_AUXILIARY_H
+
+/**
+ * @file
+ *
+ * Auxiliary Bus Interface.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/queue.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_debug.h>
+#include <rte_interrupts.h>
+#include <rte_dev.h>
+#include <rte_bus.h>
+#include <rte_kvargs.h>
+
+#define RTE_BUS_AUXILIARY_NAME "auxiliary"
+
+/* Forward declarations */
+struct rte_auxiliary_driver;
+struct rte_auxiliary_bus;
+struct rte_auxiliary_device;
+
+/**
+ * Match function for the driver to decide if device can be handled.
+ *
+ * @param name
+ *   Pointer to the auxiliary device name.
+ * @return
+ *   Whether the driver can handle the auxiliary device.
+ */
+typedef bool(rte_auxiliary_match_t)(const char *name);
+
+/**
+ * Initialization function for the driver called during auxiliary probing.
+ *
+ * @param drv
+ *   Pointer to the auxiliary driver.
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int(rte_auxiliary_probe_t)(struct rte_auxiliary_driver *drv,
+				    struct rte_auxiliary_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hotplugging.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
+				       void *addr, uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the auxiliary device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
+					 void *addr, uint64_t iova, size_t len);
+
+/**
+ * A structure describing an auxiliary device.
+ */
+struct rte_auxiliary_device {
+	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
+	struct rte_device device;                 /**< Inherit core device */
+	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
+	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
+	struct rte_auxiliary_driver *driver;      /**< Device driver */
+};
+
+/**
+ * A structure describing an auxiliary driver.
+ */
+struct rte_auxiliary_driver {
+	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
+	struct rte_driver driver;             /**< Inherit core driver. */
+	struct rte_auxiliary_bus *bus;        /**< Auxiliary bus reference. */
+	rte_auxiliary_match_t *match;         /**< Device match function. */
+	rte_auxiliary_probe_t *probe;         /**< Device probe function. */
+	rte_auxiliary_remove_t *remove;       /**< Device remove function. */
+	rte_auxiliary_dma_map_t *dma_map;     /**< Device DMA map function. */
+	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device DMA unmap function. */
+	uint32_t drv_flags;                   /**< Flags RTE_AUXILIARY_DRV_*. */
+};
+
+/**
+ * @internal
+ * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
+ */
+#define RTE_DEV_TO_AUXILIARY(ptr) \
+	container_of(ptr, struct rte_auxiliary_device, device)
+
+#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
+	container_of(ptr, const struct rte_auxiliary_device, device)
+
+#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
+	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
+
+/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
+#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
+
+/**
+ * Register an auxiliary driver.
+ *
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_experimental
+void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
+
+/** Helper for auxiliary device registration from driver instance */
+#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv) \
+	RTE_INIT(auxiliaryinitfn_ ##nm) \
+	{ \
+		(auxiliary_drv).driver.name = RTE_STR(nm); \
+		rte_auxiliary_register(&(auxiliary_drv)); \
+	} \
+	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
+
+/**
+ * Unregister an auxiliary driver.
+ *
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * @param driver
+ *   A pointer to a rte_auxiliary_driver structure describing the driver
+ *   to be unregistered.
+ */
+__rte_experimental
+void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_BUS_AUXILIARY_H */
diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
new file mode 100644
index 0000000000..a52260657c
--- /dev/null
+++ b/drivers/bus/auxiliary/version.map
@@ -0,0 +1,7 @@
+EXPERIMENTAL {
+	global:
+
+	# added in 21.08
+	rte_auxiliary_register;
+	rte_auxiliary_unregister;
+};
diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
index 410058de3a..45eab5233d 100644
--- a/drivers/bus/meson.build
+++ b/drivers/bus/meson.build
@@ -2,6 +2,7 @@
 # Copyright(c) 2017 Intel Corporation
 
 drivers = [
+        'auxiliary',
         'dpaa',
         'fslmc',
         'ifpga',
-- 
2.25.1

Re: [dpdk-dev] [PATCH v7 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

Made changes and forgot to commit :(  please ignore and refer to new v8.

> -----Original Message-----
> From: Xueming(Steven) Li <xuemingl@nvidia.com>
> Sent: Monday, July 5, 2021 1:37 PM
> Cc: dev@dpdk.org; Xueming(Steven) Li <xuemingl@nvidia.com>; Wang Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas
> Monjalon <thomas@monjalon.net>; Kinsella Ray <mdr@ashroe.eu>; Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>; Parav
> Pandit <parav@nvidia.com>; Neil Horman <nhorman@tuxdriver.com>
> Subject: [PATCH v7 2/2] bus/auxiliary: introduce auxiliary bus
> 
> Auxiliary bus [1] provides a way to split function into child-devices representing sub-domains of functionality. Each auxiliary device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> Devargs legacy syntax of auxiliary device:
>   -a auxiliary:<name>[,args...]
> Devargs generic syntax of auxiliary device:
>   -a bus=auxiliary,name=<name>/class=<class>/driver=<driver>[,args...]
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> Cc: Wang Haiyue <haiyue.wang@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>
> Cc: Kinsella Ray <mdr@ashroe.eu>
> Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> ---
>  MAINTAINERS                               |   5 +
>  doc/guides/rel_notes/release_21_08.rst    |   6 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 411 ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
>  drivers/bus/auxiliary/meson.build         |  16 +
>  drivers/bus/auxiliary/private.h           |  74 ++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
>  drivers/bus/auxiliary/version.map         |   7 +
>  drivers/bus/meson.build                   |   1 +
>  10 files changed, 921 insertions(+)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build  create mode 100644 drivers/bus/auxiliary/private.h  create mode 100644
> drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 5877a16971..9b0d0d4348 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -525,6 +525,11 @@ F: doc/guides/mempool/octeontx2.rst  Bus Drivers
>  -----------
> 
> +Auxiliary bus driver - EXPERIMENTAL
> +M: Parav Pandit <parav@nvidia.com>
> +M: Xueming Li <xuemingl@nvidia.com>
> +F: drivers/bus/auxiliary/
> +
>  Intel FPGA bus
>  M: Rosen Xu <rosen.xu@intel.com>
>  F: drivers/bus/ifpga/
> diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
> index a6ecfdf3ce..e7ef4c8a05 100644
> --- a/doc/guides/rel_notes/release_21_08.rst
> +++ b/doc/guides/rel_notes/release_21_08.rst
> @@ -55,6 +55,12 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
> 
> +* **Added auxiliary bus support.**
> +
> +  Auxiliary bus provides a way to split function into child-devices
> + representing sub-domains of functionality. Each auxiliary device
> + represents a part of its parent functionality.
> +
> 
>  Removed Items
>  -------------
> diff --git a/drivers/bus/auxiliary/auxiliary_common.c b/drivers/bus/auxiliary/auxiliary_common.c
> new file mode 100644
> index 0000000000..8a75306da5
> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_common.c
> @@ -0,0 +1,411 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates  */
> +
> +#include <string.h>
> +#include <inttypes.h>
> +#include <stdint.h>
> +#include <stdbool.h>
> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <sys/queue.h>
> +#include <rte_errno.h>
> +#include <rte_interrupts.h>
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_per_lcore.h>
> +#include <rte_memory.h>
> +#include <rte_eal.h>
> +#include <rte_eal_paging.h>
> +#include <rte_string_fns.h>
> +#include <rte_common.h>
> +#include <rte_devargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +static struct rte_devargs *
> +auxiliary_devargs_lookup(const char *name) {
> +	struct rte_devargs *devargs;
> +
> +	RTE_EAL_DEVARGS_FOREACH(RTE_BUS_AUXILIARY_NAME, devargs) {
> +		if (strcmp(devargs->name, name) == 0)
> +			return devargs;
> +	}
> +	return NULL;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist
> + *
> + * Stub for OS not supporting auxiliary bus.
> + */
> +__rte_weak bool
> +auxiliary_dev_exists(const char *name)
> +{
> +	RTE_SET_USED(name);
> +	return false;
> +}
> +
> +/*
> + * Scan the devices in the auxiliary bus.
> + *
> + * Stub for OS not supporting auxiliary bus.
> + */
> +__rte_weak int
> +auxiliary_scan(void)
> +{
> +	return 0;
> +}
> +
> +/*
> + * Update a device's devargs being scanned.
> + *
> + * @param aux_dev
> + *	AUXILIARY device.
> + */
> +void
> +auxiliary_on_scan(struct rte_auxiliary_device *aux_dev) {
> +	aux_dev->device.devargs = auxiliary_devargs_lookup(aux_dev->name);
> +}
> +
> +/*
> + * Match the auxiliary driver and device using driver function.
> + */
> +bool
> +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		const struct rte_auxiliary_device *aux_dev) {
> +	if (aux_drv->match == NULL)
> +		return false;
> +	return aux_drv->match(aux_dev->name);
> +}
> +
> +/*
> + * Call the probe() function of the driver.
> + */
> +static int
> +rte_auxiliary_probe_one_driver(struct rte_auxiliary_driver *drv,
> +			       struct rte_auxiliary_device *dev) {
> +	enum rte_iova_mode iova_mode;
> +	int ret;
> +
> +	if ((drv == NULL) || (dev == NULL))
> +		return -EINVAL;
> +
> +	/* Check if driver supports it. */
> +	if (!auxiliary_match(drv, dev))
> +		/* Match of device and driver failed */
> +		return 1;
> +
> +	/* No initialization when marked as blocked, return without error. */
> +	if (dev->device.devargs != NULL &&
> +	    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
> +		AUXILIARY_LOG(INFO, "Device is blocked, not initializing");
> +		return -1;
> +	}
> +
> +	if (dev->device.numa_node < 0) {
> +		AUXILIARY_LOG(INFO, "Device is not NUMA-aware, defaulting socket to 0");
> +		dev->device.numa_node = 0;
> +	}
> +
> +	iova_mode = rte_eal_iova_mode();
> +	if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0 &&
> +	    iova_mode != RTE_IOVA_VA) {
> +		AUXILIARY_LOG(ERR, "Driver %s expecting VA IOVA mode but current mode is PA, not initializing",
> +			      drv->driver.name);
> +		return -EINVAL;
> +	}
> +
> +	dev->driver = drv;
> +
> +	AUXILIARY_LOG(INFO, "Probe auxiliary driver: %s device: %s (socket %i)",
> +		      drv->driver.name, dev->name, dev->device.numa_node);
> +	ret = drv->probe(drv, dev);
> +	if (ret != 0)
> +		dev->driver = NULL;
> +	else
> +		dev->device.driver = &drv->driver;
> +
> +	return ret;
> +}
> +
> +/*
> + * Call the remove() function of the driver.
> + */
> +static int
> +rte_auxiliary_driver_remove_dev(struct rte_auxiliary_device *dev) {
> +	struct rte_auxiliary_driver *drv;
> +	int ret = 0;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	drv = dev->driver;
> +
> +	AUXILIARY_LOG(DEBUG, "Driver %s remove auxiliary device %s on NUMA socket %i",
> +		      drv->driver.name, dev->name, dev->device.numa_node);
> +
> +	if (drv->remove != NULL) {
> +		ret = drv->remove(dev);
> +		if (ret < 0)
> +			return ret;
> +	}
> +
> +	/* clear driver structure */
> +	dev->driver = NULL;
> +	dev->device.driver = NULL;
> +
> +	return 0;
> +}
> +
> +/*
> + * Call the probe() function of all registered driver for the given device.
> + * Return < 0 if initialization failed.
> + * Return 1 if no driver is found for this device.
> + */
> +static int
> +auxiliary_probe_all_drivers(struct rte_auxiliary_device *dev) {
> +	struct rte_auxiliary_driver *drv;
> +	int rc;
> +
> +	if (dev == NULL)
> +		return -EINVAL;
> +
> +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> +		if (!drv->match(dev->name))
> +			continue;
> +
> +		rc = rte_auxiliary_probe_one_driver(drv, dev);
> +		if (rc < 0)
> +			/* negative value is an error */
> +			return rc;
> +		if (rc > 0)
> +			/* positive value means driver doesn't support it */
> +			continue;
> +		return 0;
> +	}
> +	return 1;
> +}
> +
> +/*
> + * Scan the content of the auxiliary bus, and call the probe function
> +for
> + * all registered drivers to try to probe discovered devices.
> + */
> +static int
> +auxiliary_probe(void)
> +{
> +	struct rte_auxiliary_device *dev = NULL;
> +	size_t probed = 0, failed = 0;
> +	int ret = 0;
> +
> +	FOREACH_DEVICE_ON_AUXILIARY_BUS(dev) {
> +		probed++;
> +
> +		ret = auxiliary_probe_all_drivers(dev);
> +		if (ret < 0) {
> +			if (ret != -EEXIST) {
> +				AUXILIARY_LOG(ERR, "Requested device %s cannot be used",
> +					      dev->name);
> +				rte_errno = errno;
> +				failed++;
> +			}
> +			ret = 0;
> +		}
> +	}
> +
> +	return (probed && probed == failed) ? -1 : 0; }
> +
> +static int
> +auxiliary_parse(const char *name, void *addr) {
> +	struct rte_auxiliary_driver *drv = NULL;
> +	const char **out = addr;
> +
> +	/* Allow empty device name "auxiliary:" to bypass entire bus scan. */
> +	if (strlen(name) == 0)
> +		return 0;
> +
> +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> +		if (drv->match(name))
> +			break;
> +	}
> +	if (drv != NULL && addr != NULL)
> +		*out = name;
> +	return drv != NULL ? 0 : -1;
> +}
> +
> +/* Register a driver */
> +void
> +rte_auxiliary_register(struct rte_auxiliary_driver *driver) {
> +	TAILQ_INSERT_TAIL(&auxiliary_bus.driver_list, driver, next);
> +	driver->bus = &auxiliary_bus;
> +}
> +
> +/* Unregister a driver */
> +void
> +rte_auxiliary_unregister(struct rte_auxiliary_driver *driver) {
> +	TAILQ_REMOVE(&auxiliary_bus.driver_list, driver, next);
> +	driver->bus = NULL;
> +}
> +
> +/* Add a device to auxiliary bus */
> +void
> +auxiliary_add_device(struct rte_auxiliary_device *aux_dev) {
> +	TAILQ_INSERT_TAIL(&auxiliary_bus.device_list, aux_dev, next); }
> +
> +/* Insert a device into a predefined position in auxiliary bus */ void
> +auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			struct rte_auxiliary_device *new_aux_dev) {
> +	TAILQ_INSERT_BEFORE(exist_aux_dev, new_aux_dev, next); }
> +
> +/* Remove a device from auxiliary bus */ static void
> +rte_auxiliary_remove_device(struct rte_auxiliary_device *auxiliary_dev)
> +{
> +	TAILQ_REMOVE(&auxiliary_bus.device_list, auxiliary_dev, next); }
> +
> +static struct rte_device *
> +auxiliary_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
> +		      const void *data)
> +{
> +	const struct rte_auxiliary_device *pstart;
> +	struct rte_auxiliary_device *adev;
> +
> +	if (start != NULL) {
> +		pstart = RTE_DEV_TO_AUXILIARY_CONST(start);
> +		adev = TAILQ_NEXT(pstart, next);
> +	} else {
> +		adev = TAILQ_FIRST(&auxiliary_bus.device_list);
> +	}
> +	while (adev != NULL) {
> +		if (cmp(&adev->device, data) == 0)
> +			return &adev->device;
> +		adev = TAILQ_NEXT(adev, next);
> +	}
> +	return NULL;
> +}
> +
> +static int
> +auxiliary_plug(struct rte_device *dev)
> +{
> +	if (!auxiliary_dev_exists(dev->name))
> +		return -ENOENT;
> +	return auxiliary_probe_all_drivers(RTE_DEV_TO_AUXILIARY(dev));
> +}
> +
> +static int
> +auxiliary_unplug(struct rte_device *dev) {
> +	struct rte_auxiliary_device *adev;
> +	int ret;
> +
> +	adev = RTE_DEV_TO_AUXILIARY(dev);
> +	ret = rte_auxiliary_driver_remove_dev(adev);
> +	if (ret == 0) {
> +		rte_auxiliary_remove_device(adev);
> +		rte_devargs_remove(dev->devargs);
> +		free(adev);
> +	}
> +	return ret;
> +}
> +
> +static int
> +auxiliary_dma_map(struct rte_device *dev, void *addr, uint64_t iova,
> +size_t len) {
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || aux_dev->driver == NULL) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_map == NULL) {
> +		rte_errno = ENOTSUP;
> +		return -1;
> +	}
> +	return aux_dev->driver->dma_map(aux_dev, addr, iova, len); }
> +
> +static int
> +auxiliary_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
> +		    size_t len)
> +{
> +	struct rte_auxiliary_device *aux_dev = RTE_DEV_TO_AUXILIARY(dev);
> +
> +	if (dev == NULL || aux_dev->driver == NULL) {
> +		rte_errno = EINVAL;
> +		return -1;
> +	}
> +	if (aux_dev->driver->dma_unmap == NULL) {
> +		rte_errno = ENOTSUP;
> +		return -1;
> +	}
> +	return aux_dev->driver->dma_unmap(aux_dev, addr, iova, len); }
> +
> +bool
> +auxiliary_is_ignored_device(const char *name) {
> +	struct rte_devargs *devargs = auxiliary_devargs_lookup(name);
> +
> +	switch (auxiliary_bus.bus.conf.scan_mode) {
> +	case RTE_BUS_SCAN_ALLOWLIST:
> +		if (devargs && devargs->policy == RTE_DEV_ALLOWED)
> +			return false;
> +		break;
> +	case RTE_BUS_SCAN_UNDEFINED:
> +	case RTE_BUS_SCAN_BLOCKLIST:
> +		if (devargs == NULL || devargs->policy != RTE_DEV_BLOCKED)
> +			return false;
> +		break;
> +	}
> +	return true;
> +}
> +
> +static enum rte_iova_mode
> +auxiliary_get_iommu_class(void)
> +{
> +	const struct rte_auxiliary_driver *drv;
> +
> +	FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> +		if ((drv->drv_flags & RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA) > 0)
> +			return RTE_IOVA_VA;
> +	}
> +
> +	return RTE_IOVA_DC;
> +}
> +
> +struct rte_auxiliary_bus auxiliary_bus = {
> +	.bus = {
> +		.scan = auxiliary_scan,
> +		.probe = auxiliary_probe,
> +		.find_device = auxiliary_find_device,
> +		.plug = auxiliary_plug,
> +		.unplug = auxiliary_unplug,
> +		.parse = auxiliary_parse,
> +		.dma_map = auxiliary_dma_map,
> +		.dma_unmap = auxiliary_dma_unmap,
> +		.get_iommu_class = auxiliary_get_iommu_class,
> +		.dev_iterate = auxiliary_dev_iterate,
> +	},
> +	.device_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.device_list),
> +	.driver_list = TAILQ_HEAD_INITIALIZER(auxiliary_bus.driver_list),
> +};
> +
> +RTE_REGISTER_BUS(auxiliary, auxiliary_bus.bus);
> +RTE_LOG_REGISTER_DEFAULT(auxiliary_bus_logtype, NOTICE);
> diff --git a/drivers/bus/auxiliary/auxiliary_params.c b/drivers/bus/auxiliary/auxiliary_params.c
> new file mode 100644
> index 0000000000..cd3fa56cb4
> --- /dev/null
> +++ b/drivers/bus/auxiliary/auxiliary_params.c
> @@ -0,0 +1,59 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates  */
> +
> +#include <string.h>
> +
> +#include <rte_bus.h>
> +#include <rte_dev.h>
> +#include <rte_errno.h>
> +#include <rte_kvargs.h>
> +
> +#include "private.h"
> +#include "rte_bus_auxiliary.h"
> +
> +enum auxiliary_params {
> +	RTE_AUXILIARY_PARAM_NAME,
> +};
> +
> +static const char * const auxiliary_params_keys[] = {
> +	[RTE_AUXILIARY_PARAM_NAME] = "name",
> +};
> +
> +static int
> +auxiliary_dev_match(const struct rte_device *dev,
> +	      const void *_kvlist)
> +{
> +	const struct rte_kvargs *kvlist = _kvlist;
> +	int ret;
> +
> +	ret = rte_kvargs_process(kvlist,
> +			auxiliary_params_keys[RTE_AUXILIARY_PARAM_NAME],
> +			rte_kvargs_strcmp, (void *)(uintptr_t)dev->name);
> +
> +	return ret != 0 ? -1 : 0;
> +}
> +
> +void *
> +auxiliary_dev_iterate(const void *start,
> +		    const char *str,
> +		    const struct rte_dev_iterator *it __rte_unused) {
> +	rte_bus_find_device_t find_device;
> +	struct rte_kvargs *kvargs = NULL;
> +	struct rte_device *dev;
> +
> +	if (str != NULL) {
> +		kvargs = rte_kvargs_parse(str, auxiliary_params_keys);
> +		if (kvargs == NULL) {
> +			AUXILIARY_LOG(ERR, "cannot parse argument list %s",
> +				      str);
> +			rte_errno = EINVAL;
> +			return NULL;
> +		}
> +	}
> +	find_device = auxiliary_bus.bus.find_device;
> +	dev = find_device(start, auxiliary_dev_match, kvargs);
> +	rte_kvargs_free(kvargs);
> +	return dev;
> +}
> diff --git a/drivers/bus/auxiliary/linux/auxiliary.c b/drivers/bus/auxiliary/linux/auxiliary.c
> new file mode 100644
> index 0000000000..8464487971
> --- /dev/null
> +++ b/drivers/bus/auxiliary/linux/auxiliary.c
> @@ -0,0 +1,141 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates  */
> +
> +#include <string.h>
> +#include <dirent.h>
> +
> +#include <rte_log.h>
> +#include <rte_bus.h>
> +#include <rte_malloc.h>
> +#include <rte_devargs.h>
> +#include <rte_memcpy.h>
> +#include <eal_filesystem.h>
> +
> +#include "../rte_bus_auxiliary.h"
> +#include "../private.h"
> +
> +#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
> +
> +/* Scan one auxiliary sysfs entry, and fill the devices list from it.
> +*/ static int auxiliary_scan_one(const char *dirname, const char *name)
> +{
> +	struct rte_auxiliary_device *dev;
> +	struct rte_auxiliary_device *dev2;
> +	char filename[PATH_MAX];
> +	unsigned long tmp;
> +	int ret;
> +
> +	dev = malloc(sizeof(*dev));
> +	if (dev == NULL)
> +		return -1;
> +
> +	memset(dev, 0, sizeof(*dev));
> +	if (rte_strscpy(dev->name, name, sizeof(dev->name)) < 0) {
> +		free(dev);
> +		return -1;
> +	}
> +	dev->device.name = dev->name;
> +	dev->device.bus = &auxiliary_bus.bus;
> +
> +	/* Get NUMA node, default to 0 if not present */
> +	snprintf(filename, sizeof(filename), "%s/%s/numa_node",
> +		 dirname, name);
> +	if (access(filename, F_OK) != -1) {
> +		if (eal_parse_sysfs_value(filename, &tmp) == 0)
> +			dev->device.numa_node = tmp;
> +		else
> +			dev->device.numa_node = -1;
> +	} else {
> +		dev->device.numa_node = 0;
> +	}
> +
> +	auxiliary_on_scan(dev);
> +
> +	/* Device is valid, add in list (sorted) */
> +	TAILQ_FOREACH(dev2, &auxiliary_bus.device_list, next) {
> +		ret = strcmp(dev->name, dev2->name);
> +		if (ret > 0)
> +			continue;
> +		if (ret < 0) {
> +			auxiliary_insert_device(dev2, dev);
> +		} else { /* already registered */
> +			if (rte_dev_is_probed(&dev2->device) &&
> +			    dev2->device.devargs != dev->device.devargs) {
> +				/* To probe device with new devargs. */
> +				rte_devargs_remove(dev2->device.devargs);
> +				auxiliary_on_scan(dev2);
> +			}
> +			free(dev);
> +		}
> +		return 0;
> +	}
> +	auxiliary_add_device(dev);
> +	return 0;
> +}
> +
> +/*
> + * Test whether the auxiliary device exist  */ bool
> +auxiliary_dev_exists(const char *name) {
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +
> +	snprintf(dirname, sizeof(dirname), "%s/%s",
> +		 AUXILIARY_SYSFS_PATH, name);
> +	dir = opendir(dirname);
> +	if (dir == NULL)
> +		return false;
> +	closedir(dir);
> +	return true;
> +}
> +
> +/*
> + * Scan the devices in the auxiliary bus  */ int
> +auxiliary_scan(void)
> +{
> +	struct dirent *e;
> +	DIR *dir;
> +	char dirname[PATH_MAX];
> +	struct rte_auxiliary_driver *drv;
> +
> +	dir = opendir(AUXILIARY_SYSFS_PATH);
> +	if (dir == NULL) {
> +		AUXILIARY_LOG(INFO, "%s not found, is auxiliary module loaded?",
> +			      AUXILIARY_SYSFS_PATH);
> +		return 0;
> +	}
> +
> +	while ((e = readdir(dir)) != NULL) {
> +		if (e->d_name[0] == '.')
> +			continue;
> +
> +		if (auxiliary_is_ignored_device(e->d_name))
> +			continue;
> +
> +		snprintf(dirname, sizeof(dirname), "%s/%s",
> +			 AUXILIARY_SYSFS_PATH, e->d_name);
> +
> +		/* Ignore if no driver can handle. */
> +		FOREACH_DRIVER_ON_AUXILIARY_BUS(drv) {
> +			if (drv->match(e->d_name))
> +				break;
> +		}
> +		if (drv == NULL)
> +			continue;
> +
> +		if (auxiliary_scan_one(dirname, e->d_name) < 0)
> +			goto error;
> +	}
> +	closedir(dir);
> +	return 0;
> +
> +error:
> +	closedir(dir);
> +	return -1;
> +}
> diff --git a/drivers/bus/auxiliary/meson.build b/drivers/bus/auxiliary/meson.build
> new file mode 100644
> index 0000000000..357550eff7
> --- /dev/null
> +++ b/drivers/bus/auxiliary/meson.build
> @@ -0,0 +1,16 @@
> +# SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2021 NVIDIA
> +Corporation & Affiliates
> +
> +headers = files(
> +        'rte_bus_auxiliary.h',
> +)
> +sources = files(
> +        'auxiliary_common.c',
> +        'auxiliary_params.c',
> +)
> +if is_linux
> +    sources += files(
> +        'linux/auxiliary.c',
> +    )
> +endif
> +deps += ['kvargs']
> diff --git a/drivers/bus/auxiliary/private.h b/drivers/bus/auxiliary/private.h new file mode 100644 index 0000000000..cb3e849993
> --- /dev/null
> +++ b/drivers/bus/auxiliary/private.h
> @@ -0,0 +1,74 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates  */
> +
> +#ifndef AUXILIARY_PRIVATE_H
> +#define AUXILIARY_PRIVATE_H
> +
> +#include <stdbool.h>
> +#include <stdio.h>
> +
> +#include "rte_bus_auxiliary.h"
> +
> +extern struct rte_auxiliary_bus auxiliary_bus; extern int
> +auxiliary_bus_logtype;
> +
> +#define AUXILIARY_LOG(level, ...) \
> +	rte_log(RTE_LOG_ ## level, auxiliary_bus_logtype, \
> +		RTE_FMT("auxiliary bus: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
> +			RTE_FMT_TAIL(__VA_ARGS__,)))
> +
> +/* Auxiliary bus iterators */
> +#define FOREACH_DEVICE_ON_AUXILIARY_BUS(p) \
> +		TAILQ_FOREACH(p, &(auxiliary_bus.device_list), next)
> +
> +#define FOREACH_DRIVER_ON_AUXILIARY_BUS(p) \
> +		TAILQ_FOREACH(p, &(auxiliary_bus.driver_list), next)
> +
> +bool auxiliary_dev_exists(const char *name);
> +
> +/*
> + * Scan the content of the auxiliary bus, and the devices in the
> +devices
> + * list.
> + */
> +int auxiliary_scan(void);
> +
> +/*
> + * Update a device being scanned.
> + */
> +void auxiliary_on_scan(struct rte_auxiliary_device *aux_dev);
> +
> +/*
> + * Validate whether a device with given auxiliary device should be
> +ignored
> + * or not.
> + */
> +bool auxiliary_is_ignored_device(const char *name);
> +
> +/*
> + * Add an auxiliary device to the auxiliary bus (append to auxiliary
> +device
> + * list). This function also updates the bus references of the
> +auxiliary
> + * device and the generic device object embedded within.
> + */
> +void auxiliary_add_device(struct rte_auxiliary_device *aux_dev);
> +
> +/*
> + * Insert an auxiliary device in the auxiliary bus at a particular
> +location
> + * in the device list. It also updates the auxiliary bus reference of
> +the
> + * new devices to be inserted.
> + */
> +void auxiliary_insert_device(struct rte_auxiliary_device *exist_aux_dev,
> +			     struct rte_auxiliary_device *new_aux_dev);
> +
> +/*
> + * Match the auxiliary driver and device by driver function  */ bool
> +auxiliary_match(const struct rte_auxiliary_driver *aux_drv,
> +		     const struct rte_auxiliary_device *aux_dev);
> +
> +/*
> + * Iterate over devices, matching any device against the provided
> +string  */ void *auxiliary_dev_iterate(const void *start, const char
> +*str,
> +			    const struct rte_dev_iterator *it);
> +
> +#endif /* AUXILIARY_PRIVATE_H */
> diff --git a/drivers/bus/auxiliary/rte_bus_auxiliary.h b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> new file mode 100644
> index 0000000000..16b147e387
> --- /dev/null
> +++ b/drivers/bus/auxiliary/rte_bus_auxiliary.h
> @@ -0,0 +1,201 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates  */
> +
> +#ifndef RTE_BUS_AUXILIARY_H
> +#define RTE_BUS_AUXILIARY_H
> +
> +/**
> + * @file
> + *
> + * Auxiliary Bus Interface.
> + */
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <sys/queue.h>
> +#include <stdint.h>
> +#include <inttypes.h>
> +
> +#include <rte_debug.h>
> +#include <rte_interrupts.h>
> +#include <rte_dev.h>
> +#include <rte_bus.h>
> +#include <rte_kvargs.h>
> +
> +#define RTE_BUS_AUXILIARY_NAME "auxiliary"
> +
> +/* Forward declarations */
> +struct rte_auxiliary_driver;
> +struct rte_auxiliary_bus;
> +struct rte_auxiliary_device;
> +
> +/**
> + * Match function for the driver to decide if device can be handled.
> + *
> + * @param name
> + *   Pointer to the auxiliary device name.
> + * @return
> + *   Whether the driver can handle the auxiliary device.
> + */
> +typedef bool(rte_auxiliary_match_t)(const char *name);
> +
> +/**
> + * Initialization function for the driver called during auxiliary probing.
> + *
> + * @param drv
> + *   Pointer to the auxiliary driver.
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int(rte_auxiliary_probe_t)(struct rte_auxiliary_driver *drv,
> +				    struct rte_auxiliary_device *dev);
> +
> +/**
> + * Uninitialization function for the driver called during hotplugging.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (rte_auxiliary_remove_t)(struct rte_auxiliary_device *dev);
> +
> +/**
> + * Driver-specific DMA mapping. After a successful call the device
> + * will be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be mapped.
> + * @param iova
> + *   Starting IOVA address of memory to be mapped.
> + * @param len
> + *   Length of memory segment being mapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (rte_auxiliary_dma_map_t)(struct rte_auxiliary_device *dev,
> +				       void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * Driver-specific DMA un-mapping. After a successful call the device
> + * will not be able to read/write from/to this segment.
> + *
> + * @param dev
> + *   Pointer to the auxiliary device.
> + * @param addr
> + *   Starting virtual address of memory to be unmapped.
> + * @param iova
> + *   Starting IOVA address of memory to be unmapped.
> + * @param len
> + *   Length of memory segment being unmapped.
> + * @return
> + *   - 0 On success.
> + *   - Negative value and rte_errno is set otherwise.
> + */
> +typedef int (rte_auxiliary_dma_unmap_t)(struct rte_auxiliary_device *dev,
> +					 void *addr, uint64_t iova, size_t len);
> +
> +/**
> + * A structure describing an auxiliary device.
> + */
> +struct rte_auxiliary_device {
> +	TAILQ_ENTRY(rte_auxiliary_device) next;   /**< Next probed device. */
> +	struct rte_device device;                 /**< Inherit core device */
> +	char name[RTE_DEV_NAME_MAX_LEN + 1];      /**< ASCII device name */
> +	struct rte_intr_handle intr_handle;       /**< Interrupt handle */
> +	struct rte_auxiliary_driver *driver;      /**< Device driver */
> +};
> +
> +/** List of auxiliary devices */
> +TAILQ_HEAD(rte_auxiliary_device_list, rte_auxiliary_device);
> +/** List of auxiliary drivers */
> +TAILQ_HEAD(rte_auxiliary_driver_list, rte_auxiliary_driver);
> +
> +/**
> + * Structure describing the auxiliary bus  */ struct rte_auxiliary_bus
> +{
> +	struct rte_bus bus;                  /**< Inherit the generic class */
> +	struct rte_auxiliary_device_list device_list;  /**< List of devices */
> +	struct rte_auxiliary_driver_list driver_list;  /**< List of drivers */
> +};
> +
> +/**
> + * A structure describing an auxiliary driver.
> + */
> +struct rte_auxiliary_driver {
> +	TAILQ_ENTRY(rte_auxiliary_driver) next; /**< Next in list. */
> +	struct rte_driver driver;             /**< Inherit core driver. */
> +	struct rte_auxiliary_bus *bus;        /**< Auxiliary bus reference. */
> +	rte_auxiliary_match_t *match;         /**< Device match function. */
> +	rte_auxiliary_probe_t *probe;         /**< Device probe function. */
> +	rte_auxiliary_remove_t *remove;       /**< Device remove function. */
> +	rte_auxiliary_dma_map_t *dma_map;     /**< Device DMA map function. */
> +	rte_auxiliary_dma_unmap_t *dma_unmap; /**< Device DMA unmap function. */
> +	uint32_t drv_flags;                   /**< Flags RTE_AUXILIARY_DRV_*. */
> +};
> +
> +/**
> + * @internal
> + * Helper macro for drivers that need to convert to struct rte_auxiliary_device.
> + */
> +#define RTE_DEV_TO_AUXILIARY(ptr) \
> +	container_of(ptr, struct rte_auxiliary_device, device)
> +
> +#define RTE_DEV_TO_AUXILIARY_CONST(ptr) \
> +	container_of(ptr, const struct rte_auxiliary_device, device)
> +
> +#define RTE_ETH_DEV_TO_AUXILIARY(eth_dev) \
> +	RTE_DEV_TO_AUXILIARY((eth_dev)->device)
> +
> +/** Device driver needs IOVA as VA and cannot work with IOVA as PA */
> +#define RTE_AUXILIARY_DRV_NEED_IOVA_AS_VA 0x002
> +
> +/**
> + * Register an auxiliary driver.
> + *
> + * @param driver
> + *   A pointer to a rte_auxiliary_driver structure describing the driver
> + *   to be registered.
> + */
> +__rte_experimental
> +void rte_auxiliary_register(struct rte_auxiliary_driver *driver);
> +
> +/** Helper for auxiliary device registration from driver instance */
> +#define RTE_PMD_REGISTER_AUXILIARY(nm, auxiliary_drv) \
> +	RTE_INIT(auxiliaryinitfn_ ##nm) \
> +	{ \
> +		(auxiliary_drv).driver.name = RTE_STR(nm); \
> +		rte_auxiliary_register(&(auxiliary_drv)); \
> +	} \
> +	RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
> +
> +/**
> + * Unregister an auxiliary driver.
> + *
> + * @param driver
> + *   A pointer to a rte_auxiliary_driver structure describing the driver
> + *   to be unregistered.
> + */
> +__rte_experimental
> +void rte_auxiliary_unregister(struct rte_auxiliary_driver *driver);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* RTE_BUS_AUXILIARY_H */
> diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
> new file mode 100644
> index 0000000000..a52260657c
> --- /dev/null
> +++ b/drivers/bus/auxiliary/version.map
> @@ -0,0 +1,7 @@
> +EXPERIMENTAL {
> +	global:
> +
> +	# added in 21.08
> +	rte_auxiliary_register;
> +	rte_auxiliary_unregister;
> +};
> diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build index 410058de3a..45eab5233d 100644
> --- a/drivers/bus/meson.build
> +++ b/drivers/bus/meson.build
> @@ -2,6 +2,7 @@
>  # Copyright(c) 2017 Intel Corporation
> 
>  drivers = [
> +        'auxiliary',
>          'dpaa',
>          'fslmc',
>          'ifpga',
> --
> 2.25.1

Re: [dpdk-dev] [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus

[Andrew Rybchenko]

On 7/5/21 9:45 AM, Xueming Li wrote:
> Auxiliary bus [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> Devargs legacy syntax of auxiliary device:
>   -a auxiliary:<name>[,args...]
> Devargs generic syntax of auxiliary device:
>   -a bus=auxiliary,name=<name>/class=<class>/driver=<driver>[,args...]
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> Cc: Wang Haiyue <haiyue.wang@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>
> Cc: Kinsella Ray <mdr@ashroe.eu>
> Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>

I still don't understand if we really need to make the API
a part of stable API/ABI in the future. Can it be internal?

Re: [dpdk-dev] [PATCH v8 1/2] devargs: add common key definition

[Xueming(Steven) Li]

Sorry, this patch missed the ack line from Andrew:
	Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>


> -----Original Message-----
> From: Xueming(Steven) Li <xuemingl@nvidia.com>
> Sent: Monday, July 5, 2021 2:45 PM
> Cc: dev@dpdk.org; Xueming(Steven) Li <xuemingl@nvidia.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Matan
> Azrad <matan@nvidia.com>; Shahaf Shuler <shahafs@nvidia.com>; Slava Ovsiienko <viacheslavo@nvidia.com>; Andrew Rybchenko
> <andrew.rybchenko@oktetlabs.ru>; Chas Williams <chas3@att.com>; Min Hu (Connor) <humin29@huawei.com>; Beilei Xing
> <beilei.xing@intel.com>; Jingjing Wu <jingjing.wu@intel.com>
> Subject: [PATCH v8 1/2] devargs: add common key definition
> 
> Adds common devargs key definition for "bus", "class" and "driver".
> 
> Acked-by: Thomas Monjalon <thomas@monjalon.net>
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> ---
>  drivers/common/mlx5/mlx5_common.h       |  2 --
>  drivers/common/mlx5/mlx5_common_pci.c   |  2 +-
>  drivers/common/sfc_efx/sfc_efx.c        |  7 +++----
>  drivers/common/sfc_efx/sfc_efx.h        |  2 --
>  drivers/net/bonding/rte_eth_bond_args.c |  2 +-
>  drivers/net/i40e/i40e_ethdev_vf.c       |  5 ++---
>  drivers/net/iavf/iavf_ethdev.c          |  5 ++---
>  drivers/net/mlx5/mlx5.c                 |  4 ++--
>  drivers/net/sfc/sfc_kvargs.c            |  2 +-
>  drivers/vdpa/mlx5/mlx5_vdpa.c           |  2 +-
>  lib/eal/common/eal_common_devargs.c     | 12 ++++++------
>  lib/eal/include/rte_devargs.h           | 24 ++++++++++++++++++++++++
>  12 files changed, 43 insertions(+), 26 deletions(-)
> 
> diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
> index 1fbefe0fa6..306f2f1ab7 100644
> --- a/drivers/common/mlx5/mlx5_common.h
> +++ b/drivers/common/mlx5/mlx5_common.h
> @@ -208,8 +208,6 @@ __rte_internal
>  int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
> 
> 
> -#define MLX5_CLASS_ARG_NAME "class"
> -
>  enum mlx5_class {
>  	MLX5_CLASS_INVALID,
>  	MLX5_CLASS_NET = RTE_BIT64(0),
> diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
> index 3f16cd21cf..34747c4e07 100644
> --- a/drivers/common/mlx5/mlx5_common_pci.c
> +++ b/drivers/common/mlx5/mlx5_common_pci.c
> @@ -118,7 +118,7 @@ bus_cmdline_options_handler(__rte_unused const char *key,  static int  parse_class_options(const struct
> rte_devargs *devargs)  {
> -	const char *key = MLX5_CLASS_ARG_NAME;
> +	const char *key = RTE_DEVARGS_KEY_CLASS;
>  	struct rte_kvargs *kvlist;
>  	int ret = 0;
> 
> diff --git a/drivers/common/sfc_efx/sfc_efx.c b/drivers/common/sfc_efx/sfc_efx.c
> index 0b78933d9f..2dc5545760 100644
> --- a/drivers/common/sfc_efx/sfc_efx.c
> +++ b/drivers/common/sfc_efx/sfc_efx.c
> @@ -42,7 +42,6 @@ enum sfc_efx_dev_class  sfc_efx_dev_class_get(struct rte_devargs *devargs)  {
>  	struct rte_kvargs *kvargs;
> -	const char *key = SFC_EFX_KVARG_DEV_CLASS;
>  	enum sfc_efx_dev_class dev_class = SFC_EFX_DEV_CLASS_NET;
> 
>  	if (devargs == NULL)
> @@ -52,9 +51,9 @@ sfc_efx_dev_class_get(struct rte_devargs *devargs)
>  	if (kvargs == NULL)
>  		return dev_class;
> 
> -	if (rte_kvargs_count(kvargs, key) != 0) {
> -		rte_kvargs_process(kvargs, key, sfc_efx_kvarg_dev_class_handler,
> -				   &dev_class);
> +	if (rte_kvargs_count(kvargs, RTE_DEVARGS_KEY_CLASS) != 0) {
> +		rte_kvargs_process(kvargs, RTE_DEVARGS_KEY_CLASS,
> +				   sfc_efx_kvarg_dev_class_handler, &dev_class);
>  	}
> 
>  	rte_kvargs_free(kvargs);
> diff --git a/drivers/common/sfc_efx/sfc_efx.h b/drivers/common/sfc_efx/sfc_efx.h
> index 6b6164cb1f..c16eca60f3 100644
> --- a/drivers/common/sfc_efx/sfc_efx.h
> +++ b/drivers/common/sfc_efx/sfc_efx.h
> @@ -19,8 +19,6 @@
>  extern "C" {
>  #endif
> 
> -#define SFC_EFX_KVARG_DEV_CLASS	"class"
> -
>  enum sfc_efx_dev_class {
>  	SFC_EFX_DEV_CLASS_INVALID = 0,
>  	SFC_EFX_DEV_CLASS_NET,
> diff --git a/drivers/net/bonding/rte_eth_bond_args.c b/drivers/net/bonding/rte_eth_bond_args.c
> index 764b1b8c8e..5406e1c934 100644
> --- a/drivers/net/bonding/rte_eth_bond_args.c
> +++ b/drivers/net/bonding/rte_eth_bond_args.c
> @@ -18,7 +18,7 @@ const char *pmd_bond_init_valid_arguments[] = {
>  	PMD_BOND_SOCKET_ID_KVARG,
>  	PMD_BOND_MAC_ADDR_KVARG,
>  	PMD_BOND_AGG_MODE_KVARG,
> -	"driver",
> +	RTE_DEVARGS_KEY_DRIVER,
>  	NULL
>  };
> 
> diff --git a/drivers/net/i40e/i40e_ethdev_vf.c b/drivers/net/i40e/i40e_ethdev_vf.c
> index 385ebedcd3..0cfe13b7b2 100644
> --- a/drivers/net/i40e/i40e_ethdev_vf.c
> +++ b/drivers/net/i40e/i40e_ethdev_vf.c
> @@ -1660,7 +1660,6 @@ static int
>  i40evf_driver_selected(struct rte_devargs *devargs)  {
>  	struct rte_kvargs *kvlist;
> -	const char *key = "driver";
>  	int ret = 0;
> 
>  	if (devargs == NULL)
> @@ -1670,13 +1669,13 @@ i40evf_driver_selected(struct rte_devargs *devargs)
>  	if (kvlist == NULL)
>  		return 0;
> 
> -	if (!rte_kvargs_count(kvlist, key))
> +	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
>  		goto exit;
> 
>  	/* i40evf driver selected when there's a key-value pair:
>  	 * driver=i40evf
>  	 */
> -	if (rte_kvargs_process(kvlist, key,
> +	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
>  			       i40evf_check_driver_handler, NULL) < 0)
>  		goto exit;
> 
> diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c index 5290588b17..472538181e 100644
> --- a/drivers/net/iavf/iavf_ethdev.c
> +++ b/drivers/net/iavf/iavf_ethdev.c
> @@ -2448,7 +2448,6 @@ static int
>  iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)  {
>  	struct rte_kvargs *kvlist;
> -	const char *key = "driver";
>  	int ret = 0;
> 
>  	if (device_id != IAVF_DEV_ID_VF &&
> @@ -2464,13 +2463,13 @@ iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id)
>  	if (kvlist == NULL)
>  		return 0;
> 
> -	if (!rte_kvargs_count(kvlist, key))
> +	if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER))
>  		goto exit;
> 
>  	/* i40evf driver selected when there's a key-value pair:
>  	 * driver=i40evf
>  	 */
> -	if (rte_kvargs_process(kvlist, key,
> +	if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER,
>  			       iavf_drv_i40evf_check_handler, NULL) < 0)
>  		goto exit;
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index cf1815cb74..d0faa45944 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -1931,7 +1931,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
>  		config->max_dump_files_num = tmp;
>  	} else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
>  		config->lro.timeout = tmp;
> -	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
> +	} else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) {
>  		DRV_LOG(DEBUG, "class argument is %s.", val);
>  	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
>  		config->log_hp_size = tmp;
> @@ -2002,7 +2002,7 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
>  		MLX5_REPRESENTOR,
>  		MLX5_MAX_DUMP_FILES_NUM,
>  		MLX5_LRO_TIMEOUT_USEC,
> -		MLX5_CLASS_ARG_NAME,
> +		RTE_DEVARGS_KEY_CLASS,
>  		MLX5_HP_BUF_SIZE,
>  		MLX5_RECLAIM_MEM,
>  		MLX5_SYS_MEM_EN,
> diff --git a/drivers/net/sfc/sfc_kvargs.c b/drivers/net/sfc/sfc_kvargs.c index 0efa92ed28..974c05e68e 100644
> --- a/drivers/net/sfc/sfc_kvargs.c
> +++ b/drivers/net/sfc/sfc_kvargs.c
> @@ -28,7 +28,7 @@ sfc_kvargs_parse(struct sfc_adapter *sa)
>  		SFC_KVARG_TX_DATAPATH,
>  		SFC_KVARG_FW_VARIANT,
>  		SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
> -		SFC_EFX_KVARG_DEV_CLASS,
> +		RTE_DEVARGS_KEY_CLASS,
>  		NULL,
>  	};
> 
> diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c index e5e03e6582..8b5bfd8c3d 100644
> --- a/drivers/vdpa/mlx5/mlx5_vdpa.c
> +++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
> @@ -588,7 +588,7 @@ mlx5_vdpa_args_check_handler(const char *key, const char *val, void *opaque)
>  	unsigned long tmp;
>  	int n_cores = sysconf(_SC_NPROCESSORS_ONLN);
> 
> -	if (strcmp(key, "class") == 0)
> +	if (strcmp(key, RTE_DEVARGS_KEY_CLASS) == 0)
>  		return 0;
>  	errno = 0;
>  	tmp = strtoul(val, NULL, 0);
> diff --git a/lib/eal/common/eal_common_devargs.c b/lib/eal/common/eal_common_devargs.c
> index b31ac879a9..23aaf8b7e4 100644
> --- a/lib/eal/common/eal_common_devargs.c
> +++ b/lib/eal/common/eal_common_devargs.c
> @@ -49,9 +49,9 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
>  		const char *str;
>  		struct rte_kvargs *kvlist;
>  	} layers[] = {
> -		{ "bus=",    NULL, NULL, },
> -		{ "class=",  NULL, NULL, },
> -		{ "driver=", NULL, NULL, },
> +		{ RTE_DEVARGS_KEY_BUS "=",    NULL, NULL, },
> +		{ RTE_DEVARGS_KEY_CLASS "=",  NULL, NULL, },
> +		{ RTE_DEVARGS_KEY_DRIVER "=", NULL, NULL, },
>  	};
>  	struct rte_kvargs_pair *kv = NULL;
>  	struct rte_class *cls = NULL;
> @@ -118,7 +118,7 @@ rte_devargs_layers_parse(struct rte_devargs *devargs,
>  		if (layers[i].kvlist == NULL)
>  			continue;
>  		kv = &layers[i].kvlist->pairs[0];
> -		if (strcmp(kv->key, "bus") == 0) {
> +		if (strcmp(kv->key, RTE_DEVARGS_KEY_BUS) == 0) {
>  			bus = rte_bus_find_by_name(kv->value);
>  			if (bus == NULL) {
>  				RTE_LOG(ERR, EAL, "Could not find bus \"%s\"\n", @@ -126,7 +126,7 @@
> rte_devargs_layers_parse(struct rte_devargs *devargs,
>  				ret = -EFAULT;
>  				goto get_out;
>  			}
> -		} else if (strcmp(kv->key, "class") == 0) {
> +		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_CLASS) == 0) {
>  			cls = rte_class_find_by_name(kv->value);
>  			if (cls == NULL) {
>  				RTE_LOG(ERR, EAL, "Could not find class \"%s\"\n", @@ -134,7 +134,7 @@
> rte_devargs_layers_parse(struct rte_devargs *devargs,
>  				ret = -EFAULT;
>  				goto get_out;
>  			}
> -		} else if (strcmp(kv->key, "driver") == 0) {
> +		} else if (strcmp(kv->key, RTE_DEVARGS_KEY_DRIVER) == 0) {
>  			/* Ignore */
>  			continue;
>  		}
> diff --git a/lib/eal/include/rte_devargs.h b/lib/eal/include/rte_devargs.h index 1e595b3c51..cd90944fe8 100644
> --- a/lib/eal/include/rte_devargs.h
> +++ b/lib/eal/include/rte_devargs.h
> @@ -25,6 +25,30 @@ extern "C" {
>  #include <rte_compat.h>
>  #include <rte_bus.h>
> 
> +/**
> + * Bus type key in global devargs syntax.
> + *
> + * Legacy devargs parser doesn't use this key as bus type
> + * is resolved as first optional value separated by ":".
> + */
> +#define RTE_DEVARGS_KEY_BUS "bus"
> +
> +/**
> + * Class type key in global devargs syntax.
> + *
> + * Legacy devargs parser doesn't parse class type. PMD driver is
> + * encouraged to use this key to resolve class type.
> + */
> +#define RTE_DEVARGS_KEY_CLASS "class"
> +
> +/**
> + * Driver type key in global devargs syntax.
> + *
> + * Legacy devargs parser doesn't parse driver type. PMD driver is
> + * encouraged to use this key to resolve driver type.
> + */
> +#define RTE_DEVARGS_KEY_DRIVER "driver"
> +
>  /**
>   * Type of generic device
>   */
> --
> 2.25.1

Re: [dpdk-dev] [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

Hi Andrew,

> -----Original Message-----
> From: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> Sent: Monday, July 5, 2021 5:19 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Kinsella Ray
> <mdr@ashroe.eu>; Parav Pandit <parav@nvidia.com>; Neil Horman <nhorman@tuxdriver.com>
> Subject: Re: [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus
> 
> On 7/5/21 9:45 AM, Xueming Li wrote:
> > Auxiliary bus [1] provides a way to split function into child-devices
> > representing sub-domains of functionality. Each auxiliary device
> > represents a part of its parent functionality.
> >
> > Auxiliary device is identified by unique device name, sysfs path:
> >   /sys/bus/auxiliary/devices/<name>
> >
> > Devargs legacy syntax of auxiliary device:
> >   -a auxiliary:<name>[,args...]
> > Devargs generic syntax of auxiliary device:
> >   -a bus=auxiliary,name=<name>/class=<class>/driver=<driver>[,args...]
> >
> > [1] kernel auxiliary bus document:
> > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > Cc: Wang Haiyue <haiyue.wang@intel.com>
> > Cc: Thomas Monjalon <thomas@monjalon.net>
> > Cc: Kinsella Ray <mdr@ashroe.eu>
> > Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> 
> I still don't understand if we really need to make the API a part of stable API/ABI in the future. Can it be internal?

There was some discussion on this with Thomas in earlier version. Users might want to register/unregister their own PMD driver,
Is this a valid scenario?

Re: [dpdk-dev] [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus

[Andrew Rybchenko]

On 7/5/21 12:30 PM, Xueming(Steven) Li wrote:
> Hi Andrew,
> 
>> -----Original Message-----
>> From: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
>> Sent: Monday, July 5, 2021 5:19 PM
>> To: Xueming(Steven) Li <xuemingl@nvidia.com>
>> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Kinsella Ray
>> <mdr@ashroe.eu>; Parav Pandit <parav@nvidia.com>; Neil Horman <nhorman@tuxdriver.com>
>> Subject: Re: [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus
>>
>> On 7/5/21 9:45 AM, Xueming Li wrote:
>>> Auxiliary bus [1] provides a way to split function into child-devices
>>> representing sub-domains of functionality. Each auxiliary device
>>> represents a part of its parent functionality.
>>>
>>> Auxiliary device is identified by unique device name, sysfs path:
>>>   /sys/bus/auxiliary/devices/<name>
>>>
>>> Devargs legacy syntax of auxiliary device:
>>>   -a auxiliary:<name>[,args...]
>>> Devargs generic syntax of auxiliary device:
>>>   -a bus=auxiliary,name=<name>/class=<class>/driver=<driver>[,args...]
>>>
>>> [1] kernel auxiliary bus document:
>>> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
>>>
>>> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
>>> Cc: Wang Haiyue <haiyue.wang@intel.com>
>>> Cc: Thomas Monjalon <thomas@monjalon.net>
>>> Cc: Kinsella Ray <mdr@ashroe.eu>
>>> Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
>>
>> I still don't understand if we really need to make the API a part of stable API/ABI in the future. Can it be internal?
> 
> There was some discussion on this with Thomas in earlier version.
> Users might want to register/unregister their own PMD driver,
> Is this a valid scenario?

Yes, it is true, but should DPDK care that much about
out-of-tree drivers. I'm just asking since don't know
techboard position on it.

Re: [dpdk-dev] [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

05/07/2021 11:35, Andrew Rybchenko:
> On 7/5/21 12:30 PM, Xueming(Steven) Li wrote:
> > From: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
> >> I still don't understand if we really need to make the API a part of stable API/ABI in the future. Can it be internal?
> > 
> > There was some discussion on this with Thomas in earlier version.
> > Users might want to register/unregister their own PMD driver,
> > Is this a valid scenario?
> 
> Yes, it is true, but should DPDK care that much about
> out-of-tree drivers. I'm just asking since don't know
> techboard position on it.

I think there is a consensus to allow out-of-tree drivers
without any compatibility commitment.

Some other bus drivers are exporting some API like in this patch.
We could discuss again in techboard what to make internal.
If it is decided to hide buses API, we could change all bus drivers
later in DPDK 21.11.

Re: [dpdk-dev] [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus

[Andrew Rybchenko]

On 7/5/21 5:57 PM, Thomas Monjalon wrote:
> 05/07/2021 11:35, Andrew Rybchenko:
>> On 7/5/21 12:30 PM, Xueming(Steven) Li wrote:
>>> From: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
>>>> I still don't understand if we really need to make the API a part of stable API/ABI in the future. Can it be internal?
>>>
>>> There was some discussion on this with Thomas in earlier version.
>>> Users might want to register/unregister their own PMD driver,
>>> Is this a valid scenario?
>>
>> Yes, it is true, but should DPDK care that much about
>> out-of-tree drivers. I'm just asking since don't know
>> techboard position on it.
> 
> I think there is a consensus to allow out-of-tree drivers
> without any compatibility commitment.
> 
> Some other bus drivers are exporting some API like in this patch.
> We could discuss again in techboard what to make internal.
> If it is decided to hide buses API, we could change all bus drivers
> later in DPDK 21.11.

OK, thanks.

Re: [dpdk-dev] [PATCH v8 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

05/07/2021 08:45, Xueming Li:
> Auxiliary bus [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> Devargs legacy syntax of auxiliary device:
>   -a auxiliary:<name>[,args...]
> Devargs generic syntax of auxiliary device:
>   -a bus=auxiliary,name=<name>/class=<class>/driver=<driver>[,args...]
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> Cc: Wang Haiyue <haiyue.wang@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>
> Cc: Kinsella Ray <mdr@ashroe.eu>
> Acked-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>

Acked-by: Thomas Monjalon <thomas@monjalon.net>

Applied, thanks

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Kinsella, Ray]

On 25/06/2021 12:47, Xueming Li wrote:
> Auxiliary bus [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> Devargs legacy syntax ofauxiliary device:
>   -a auxiliary:<name>[,args...]
> Devargs generic syntax of auxiliary device:
>   -a bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> Cc: Wang Haiyue <haiyue.wang@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>
> Cc: Kinsella Ray <mdr@ashroe.eu>
> ---
>  MAINTAINERS                               |   5 +
>  doc/guides/rel_notes/release_21_08.rst    |   6 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 411 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
>  drivers/bus/auxiliary/meson.build         |  16 +
>  drivers/bus/auxiliary/private.h           |  74 ++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
>  drivers/bus/auxiliary/version.map         |   7 +
>  drivers/bus/meson.build                   |   1 +
>  10 files changed, 921 insertions(+)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 

[SNIP]

> diff --git a/drivers/bus/auxiliary/version.map b/drivers/bus/auxiliary/version.map
> new file mode 100644
> index 0000000000..a52260657c
> --- /dev/null
> +++ b/drivers/bus/auxiliary/version.map
> @@ -0,0 +1,7 @@
> +EXPERIMENTAL {
> +	global:
> +
> +	# added in 21.08
> +	rte_auxiliary_register;
> +	rte_auxiliary_unregister;
> +};

Aren't these really internal?

> diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build
> index 410058de3a..45eab5233d 100644
> --- a/drivers/bus/meson.build
> +++ b/drivers/bus/meson.build
> @@ -2,6 +2,7 @@
>  # Copyright(c) 2017 Intel Corporation
>  
>  drivers = [
> +        'auxiliary',
>          'dpaa',
>          'fslmc',
>          'ifpga',
> 

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Kinsella, Ray <mdr@ashroe.eu>
> Sent: Wednesday, August 4, 2021 5:50 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>; NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Neil Horman
> <nhorman@tuxdriver.com>
> Subject: Re: [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus
> 
> 
> 
> On 25/06/2021 12:47, Xueming Li wrote:
> > Auxiliary bus [1] provides a way to split function into child-devices
> > representing sub-domains of functionality. Each auxiliary device
> > represents a part of its parent functionality.
> >
> > Auxiliary device is identified by unique device name, sysfs path:
> >   /sys/bus/auxiliary/devices/<name>
> >
> > Devargs legacy syntax ofauxiliary device:
> >   -a auxiliary:<name>[,args...]
> > Devargs generic syntax of auxiliary device:
> >   -a bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,
> >
> > [1] kernel auxiliary bus document:
> > https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> >
> > Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> > Cc: Wang Haiyue <haiyue.wang@intel.com>
> > Cc: Thomas Monjalon <thomas@monjalon.net>
> > Cc: Kinsella Ray <mdr@ashroe.eu>
> > ---
> >  MAINTAINERS                               |   5 +
> >  doc/guides/rel_notes/release_21_08.rst    |   6 +
> >  drivers/bus/auxiliary/auxiliary_common.c  | 411
> > ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
> >  drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
> >  drivers/bus/auxiliary/meson.build         |  16 +
> >  drivers/bus/auxiliary/private.h           |  74 ++++
> >  drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
> >  drivers/bus/auxiliary/version.map         |   7 +
> >  drivers/bus/meson.build                   |   1 +
> >  10 files changed, 921 insertions(+)
> >  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
> >  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> >  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> >  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> > 100644 drivers/bus/auxiliary/private.h  create mode 100644
> > drivers/bus/auxiliary/rte_bus_auxiliary.h
> >  create mode 100644 drivers/bus/auxiliary/version.map
> >
> 
> [SNIP]
> 
> > diff --git a/drivers/bus/auxiliary/version.map
> > b/drivers/bus/auxiliary/version.map
> > new file mode 100644
> > index 0000000000..a52260657c
> > --- /dev/null
> > +++ b/drivers/bus/auxiliary/version.map
> > @@ -0,0 +1,7 @@
> > +EXPERIMENTAL {
> > +	global:
> > +
> > +	# added in 21.08
> > +	rte_auxiliary_register;
> > +	rte_auxiliary_unregister;
> > +};
> 
> Aren't these really internal?

Hi Kinsella,
Application might implement driver, so keep it public, same as rte_pci_register.

> 
> > diff --git a/drivers/bus/meson.build b/drivers/bus/meson.build index
> > 410058de3a..45eab5233d 100644
> > --- a/drivers/bus/meson.build
> > +++ b/drivers/bus/meson.build
> > @@ -2,6 +2,7 @@
> >  # Copyright(c) 2017 Intel Corporation
> >
> >  drivers = [
> > +        'auxiliary',
> >          'dpaa',
> >          'fslmc',
> >          'ifpga',
> >

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Kinsella, Ray]

On 25/06/2021 12:47, Xueming Li wrote:
> Auxiliary bus [1] provides a way to split function into child-devices
> representing sub-domains of functionality. Each auxiliary device
> represents a part of its parent functionality.
> 
> Auxiliary device is identified by unique device name, sysfs path:
>   /sys/bus/auxiliary/devices/<name>
> 
> Devargs legacy syntax ofauxiliary device:
>   -a auxiliary:<name>[,args...]
> Devargs generic syntax of auxiliary device:
>   -a bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,
> 
> [1] kernel auxiliary bus document:
> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
> 
> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> Cc: Wang Haiyue <haiyue.wang@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>
> Cc: Kinsella Ray <mdr@ashroe.eu>
> ---
>  MAINTAINERS                               |   5 +
>  doc/guides/rel_notes/release_21_08.rst    |   6 +
>  drivers/bus/auxiliary/auxiliary_common.c  | 411 ++++++++++++++++++++++
>  drivers/bus/auxiliary/auxiliary_params.c  |  59 ++++
>  drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
>  drivers/bus/auxiliary/meson.build         |  16 +
>  drivers/bus/auxiliary/private.h           |  74 ++++
>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
>  drivers/bus/auxiliary/version.map         |   7 +
>  drivers/bus/meson.build                   |   1 +
>  10 files changed, 921 insertions(+)
>  create mode 100644 drivers/bus/auxiliary/auxiliary_common.c
>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>  create mode 100644 drivers/bus/auxiliary/meson.build
>  create mode 100644 drivers/bus/auxiliary/private.h
>  create mode 100644 drivers/bus/auxiliary/rte_bus_auxiliary.h
>  create mode 100644 drivers/bus/auxiliary/version.map
> 

Acked-by: Ray Kinsella <mdr@ashroe.eu>

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Kinsella, Ray]

On 04/08/2021 13:11, Xueming(Steven) Li wrote:
> 
> 
>> -----Original Message-----
>> From: Kinsella, Ray <mdr@ashroe.eu>
>> Sent: Wednesday, August 4, 2021 7:46 PM
>> To: Xueming(Steven) Li <xuemingl@nvidia.com>
>> Subject: Re: [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus
>>
>>
>>
>> On 04/08/2021 12:21, Xueming(Steven) Li wrote:
>>>
>>>
>>>> -----Original Message-----
>>>> From: Kinsella, Ray <mdr@ashroe.eu>
>>>> Sent: Wednesday, August 4, 2021 6:00 PM
>>>> To: Xueming(Steven) Li <xuemingl@nvidia.com>
>>>> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>;
>>>> NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Neil Horman
>>>> <nhorman@tuxdriver.com>
>>>> Subject: Re: [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus
>>>>
>>>>
>>>>
>>>> On 25/06/2021 12:47, Xueming Li wrote:
>>>>> Auxiliary bus [1] provides a way to split function into
>>>>> child-devices representing sub-domains of functionality. Each
>>>>> auxiliary device represents a part of its parent functionality.
>>>>>
>>>>> Auxiliary device is identified by unique device name, sysfs path:
>>>>>   /sys/bus/auxiliary/devices/<name>
>>>>>
>>>>> Devargs legacy syntax ofauxiliary device:
>>>>>   -a auxiliary:<name>[,args...]
>>>>> Devargs generic syntax of auxiliary device:
>>>>>   -a bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,
>>>>>
>>>>> [1] kernel auxiliary bus document:
>>>>> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.html
>>>>>
>>>>> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
>>>>> Cc: Wang Haiyue <haiyue.wang@intel.com>
>>>>> Cc: Thomas Monjalon <thomas@monjalon.net>
>>>>> Cc: Kinsella Ray <mdr@ashroe.eu>
>>>>> ---
>>>>>  MAINTAINERS                               |   5 +
>>>>>  doc/guides/rel_notes/release_21_08.rst    |   6 +
>>>>>  drivers/bus/auxiliary/auxiliary_common.c  | 411
>>>>> ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c  |
>>>>> ++++++++++++++++++++++ 59 ++++
>>>>>  drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
>>>>>  drivers/bus/auxiliary/meson.build         |  16 +
>>>>>  drivers/bus/auxiliary/private.h           |  74 ++++
>>>>>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
>>>>>  drivers/bus/auxiliary/version.map         |   7 +
>>>>>  drivers/bus/meson.build                   |   1 +
>>>>>  10 files changed, 921 insertions(+)  create mode 100644
>>>>> drivers/bus/auxiliary/auxiliary_common.c
>>>>>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
>>>>>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
>>>>>  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
>>>>> 100644 drivers/bus/auxiliary/private.h  create mode 100644
>>>>> drivers/bus/auxiliary/rte_bus_auxiliary.h
>>>>>  create mode 100644 drivers/bus/auxiliary/version.map
>>>>>
>>>>
>>>> Acked-by: Ray Kinsella <mdr@ashroe.eu>
>>>
>>> Thanks, but this patch already integrated :)
>>
>> It appears in the order in which I am going through my email is incorrect. :-)
>>
>>>
>>> Would you like to have a look at another deprecation notice? Andrew reviewed RFC:
>>> https://mails.dpdk.org/archives/dev/2021-August/216007.html
>>>
>>
>> Its not strictly a depreciation notice though, you are not breaking anything right.
>> Since you are not breaking anything, don't think the notice is required in the 21.11 timeframe.
>>
>> Now if you where doing it in 21.08, it would be an ABI change and that would be a different story.
> 
> Thanks for looking at this!
> Yes, it targets to 21.11. The offloading flag is fine, but the shared_group does break ABI, detail:
> 	https://mails.dpdk.org/archives/dev/2021-July/215575.html

Right ... its a new field, not a depreciation as such.
What I mean by this is that no existing code is broken.

21.11 is a new ABI in any case and you are not depreciating anything, so no notice is required. 

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Xueming(Steven) Li]

> -----Original Message-----
> From: Kinsella, Ray <mdr@ashroe.eu>
> Sent: Wednesday, August 4, 2021 8:14 PM
> To: Xueming(Steven) Li <xuemingl@nvidia.com>; dpdk-dev <dev@dpdk.org>
> Subject: Re: [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus
> 
> 
> 
> On 04/08/2021 13:11, Xueming(Steven) Li wrote:
> >
> >
> >> -----Original Message-----
> >> From: Kinsella, Ray <mdr@ashroe.eu>
> >> Sent: Wednesday, August 4, 2021 7:46 PM
> >> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> >> Subject: Re: [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus
> >>
> >>
> >>
> >> On 04/08/2021 12:21, Xueming(Steven) Li wrote:
> >>>
> >>>
> >>>> -----Original Message-----
> >>>> From: Kinsella, Ray <mdr@ashroe.eu>
> >>>> Sent: Wednesday, August 4, 2021 6:00 PM
> >>>> To: Xueming(Steven) Li <xuemingl@nvidia.com>
> >>>> Cc: dev@dpdk.org; Wang Haiyue <haiyue.wang@intel.com>;
> >>>> NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Neil Horman
> >>>> <nhorman@tuxdriver.com>
> >>>> Subject: Re: [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus
> >>>>
> >>>>
> >>>>
> >>>> On 25/06/2021 12:47, Xueming Li wrote:
> >>>>> Auxiliary bus [1] provides a way to split function into
> >>>>> child-devices representing sub-domains of functionality. Each
> >>>>> auxiliary device represents a part of its parent functionality.
> >>>>>
> >>>>> Auxiliary device is identified by unique device name, sysfs path:
> >>>>>   /sys/bus/auxiliary/devices/<name>
> >>>>>
> >>>>> Devargs legacy syntax ofauxiliary device:
> >>>>>   -a auxiliary:<name>[,args...]
> >>>>> Devargs generic syntax of auxiliary device:
> >>>>>   -a
> >>>>> bus=auxiliary,name=<name>,,/class=<classs>,,/driver=<driver>,,
> >>>>>
> >>>>> [1] kernel auxiliary bus document:
> >>>>> https://www.kernel.org/doc/html/latest/driver-api/auxiliary_bus.ht
> >>>>> ml
> >>>>>
> >>>>> Signed-off-by: Xueming Li <xuemingl@nvidia.com>
> >>>>> Cc: Wang Haiyue <haiyue.wang@intel.com>
> >>>>> Cc: Thomas Monjalon <thomas@monjalon.net>
> >>>>> Cc: Kinsella Ray <mdr@ashroe.eu>
> >>>>> ---
> >>>>>  MAINTAINERS                               |   5 +
> >>>>>  doc/guides/rel_notes/release_21_08.rst    |   6 +
> >>>>>  drivers/bus/auxiliary/auxiliary_common.c  | 411
> >>>>> ++++++++++++++++++++++  drivers/bus/auxiliary/auxiliary_params.c
> >>>>> ++++++++++++++++++++++ |
> >>>>> ++++++++++++++++++++++ 59 ++++
> >>>>>  drivers/bus/auxiliary/linux/auxiliary.c   | 141 ++++++++
> >>>>>  drivers/bus/auxiliary/meson.build         |  16 +
> >>>>>  drivers/bus/auxiliary/private.h           |  74 ++++
> >>>>>  drivers/bus/auxiliary/rte_bus_auxiliary.h | 201 +++++++++++
> >>>>>  drivers/bus/auxiliary/version.map         |   7 +
> >>>>>  drivers/bus/meson.build                   |   1 +
> >>>>>  10 files changed, 921 insertions(+)  create mode 100644
> >>>>> drivers/bus/auxiliary/auxiliary_common.c
> >>>>>  create mode 100644 drivers/bus/auxiliary/auxiliary_params.c
> >>>>>  create mode 100644 drivers/bus/auxiliary/linux/auxiliary.c
> >>>>>  create mode 100644 drivers/bus/auxiliary/meson.build  create mode
> >>>>> 100644 drivers/bus/auxiliary/private.h  create mode 100644
> >>>>> drivers/bus/auxiliary/rte_bus_auxiliary.h
> >>>>>  create mode 100644 drivers/bus/auxiliary/version.map
> >>>>>
> >>>>
> >>>> Acked-by: Ray Kinsella <mdr@ashroe.eu>
> >>>
> >>> Thanks, but this patch already integrated :)
> >>
> >> It appears in the order in which I am going through my email is
> >> incorrect. :-)
> >>
> >>>
> >>> Would you like to have a look at another deprecation notice? Andrew reviewed RFC:
> >>> https://mails.dpdk.org/archives/dev/2021-August/216007.html
> >>>
> >>
> >> Its not strictly a depreciation notice though, you are not breaking anything right.
> >> Since you are not breaking anything, don't think the notice is required in the 21.11 timeframe.
> >>
> >> Now if you where doing it in 21.08, it would be an ABI change and that would be a different story.
> >
> > Thanks for looking at this!
> > Yes, it targets to 21.11. The offloading flag is fine, but the shared_group does break ABI, detail:
> > 	https://mails.dpdk.org/archives/dev/2021-July/215575.html
> 
> Right ... its a new field, not a depreciation as such.
> What I mean by this is that no existing code is broken.
> 
> 21.11 is a new ABI in any case and you are not depreciating anything, so no notice is required.

Maybe it a new process, confirmed with Thomas, it's expected:
https://doc.dpdk.org/guides/contributing/abi_policy.html#abi-changes

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

04/08/2021 15:00, Xueming(Steven) Li:
> From: Kinsella, Ray <mdr@ashroe.eu>
> > On 04/08/2021 13:11, Xueming(Steven) Li wrote:
> > > From: Kinsella, Ray <mdr@ashroe.eu>
> > >> Its not strictly a depreciation notice though, you are not breaking anything right.
> > >> Since you are not breaking anything, don't think the notice is required in the 21.11 timeframe.
> > >>
> > >> Now if you where doing it in 21.08, it would be an ABI change and that would be a different story.
> > >
> > > Thanks for looking at this!
> > > Yes, it targets to 21.11. The offloading flag is fine, but the shared_group does break ABI, detail:
> > > 	https://mails.dpdk.org/archives/dev/2021-July/215575.html
> > 
> > Right ... its a new field, not a depreciation as such.
> > What I mean by this is that no existing code is broken.
> > 
> > 21.11 is a new ABI in any case and you are not depreciating anything, so no notice is required.
> 
> Maybe it a new process, confirmed with Thomas, it's expected:
> https://doc.dpdk.org/guides/contributing/abi_policy.html#abi-changes

I think what Ray means is that it breaks ABI but not API,
so he doesn't consider a notice is required.
My understanding of the policy is that *any* ABI change requires a notice.
But if you want to make it lighter and allow any non-announced ABI change
in an ABI-breaking release, I think I would vote for.

Cc techboard@dpdk.org

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Kinsella, Ray]

On 04/08/2021 14:12, Thomas Monjalon wrote:
> 04/08/2021 15:00, Xueming(Steven) Li:
>> From: Kinsella, Ray <mdr@ashroe.eu>
>>> On 04/08/2021 13:11, Xueming(Steven) Li wrote:
>>>> From: Kinsella, Ray <mdr@ashroe.eu>
>>>>> Its not strictly a depreciation notice though, you are not breaking anything right.
>>>>> Since you are not breaking anything, don't think the notice is required in the 21.11 timeframe.
>>>>>
>>>>> Now if you where doing it in 21.08, it would be an ABI change and that would be a different story.
>>>>
>>>> Thanks for looking at this!
>>>> Yes, it targets to 21.11. The offloading flag is fine, but the shared_group does break ABI, detail:
>>>> 	https://mails.dpdk.org/archives/dev/2021-July/215575.html
>>>
>>> Right ... its a new field, not a depreciation as such.
>>> What I mean by this is that no existing code is broken.
>>>
>>> 21.11 is a new ABI in any case and you are not depreciating anything, so no notice is required.
>>
>> Maybe it a new process, confirmed with Thomas, it's expected:
>> https://doc.dpdk.org/guides/contributing/abi_policy.html#abi-changes
> 
> I think what Ray means is that it breaks ABI but not API,
> so he doesn't consider a notice is required.

> My understanding of the policy is that *any* ABI change requires a notice.
> But if you want to make it lighter and allow any non-announced ABI change
> in an ABI-breaking release, I think I would vote for.

Thanks for clarifying Thomas ... you are correct. 

> 
> Cc techboard@dpdk.org
> 
 

Re: [dpdk-dev] [PATCH v6 2/2] bus/auxiliary: introduce auxiliary bus

[Thomas Monjalon]

04/08/2021 15:53, Kinsella, Ray:
> On 04/08/2021 14:12, Thomas Monjalon wrote:
> > 04/08/2021 15:00, Xueming(Steven) Li:
> >> From: Kinsella, Ray <mdr@ashroe.eu>
> >>> On 04/08/2021 13:11, Xueming(Steven) Li wrote:
> >>>> From: Kinsella, Ray <mdr@ashroe.eu>
> >>>>> Its not strictly a depreciation notice though, you are not breaking anything right.
> >>>>> Since you are not breaking anything, don't think the notice is required in the 21.11 timeframe.
> >>>>>
> >>>>> Now if you where doing it in 21.08, it would be an ABI change and that would be a different story.
> >>>>
> >>>> Thanks for looking at this!
> >>>> Yes, it targets to 21.11. The offloading flag is fine, but the shared_group does break ABI, detail:
> >>>> 	https://mails.dpdk.org/archives/dev/2021-July/215575.html
> >>>
> >>> Right ... its a new field, not a depreciation as such.
> >>> What I mean by this is that no existing code is broken.
> >>>
> >>> 21.11 is a new ABI in any case and you are not depreciating anything, so no notice is required.
> >>
> >> Maybe it a new process, confirmed with Thomas, it's expected:
> >> https://doc.dpdk.org/guides/contributing/abi_policy.html#abi-changes
> > 
> > I think what Ray means is that it breaks ABI but not API,
> > so he doesn't consider a notice is required.
> 
> > My understanding of the policy is that *any* ABI change requires a notice.
> > But if you want to make it lighter and allow any non-announced ABI change
> > in an ABI-breaking release, I think I would vote for.
> 
> Thanks for clarifying Thomas ... you are correct.

In the meantime, let's review and ack notices, even if ABI-only change:
https://patches.dpdk.org/bundle/tmonjalo/deprecation-notices/

We'll discuss later if we can accept more ABI change,
but we should try to be on the safe side for those already announced.