[dpdk-dev] [RFC 11/14] common/mlx5: support auxiliary bus

[Xueming Li <xuemingl@nvidia.com>]

This patch adds auxiliary bus driver and delegate to
registered internal mlx5 common device drivers, i.e. eth, vdpa...

Current major target is to support SubFunction on auxiliary bus.

As a limitation of current driver, numa node of device is detected from
PCI bus of device symbol link, will remove once numa node file available
on sysfs.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/linux/meson.build         |   3 +
 .../common/mlx5/linux/mlx5_common_auxiliary.c | 173 ++++++++++++++++++
 drivers/common/mlx5/linux/mlx5_common_verbs.c |   5 +-
 drivers/common/mlx5/mlx5_common.c             |   3 +
 drivers/common/mlx5/mlx5_common.h             |   5 +
 drivers/common/mlx5/mlx5_common_private.h     |   5 +
 drivers/common/mlx5/version.map               |   2 +
 7 files changed, 195 insertions(+), 1 deletion(-)
 create mode 100644 drivers/common/mlx5/linux/mlx5_common_auxiliary.c

diff --git a/drivers/common/mlx5/linux/meson.build b/drivers/common/mlx5/linux/meson.build
index 007834a49b..a1070acb77 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -48,10 +48,13 @@ endif
 sources += files('mlx5_nl.c')
 sources += files('mlx5_common_os.c')
 sources += files('mlx5_common_verbs.c')
+sources += files('mlx5_common_auxiliary.c')
 if not dlopen_ibverbs
     sources += files('mlx5_glue.c')
 endif
 
+deps += ['bus_auxiliary']
+
 # To maintain the compatibility with the make build system
 # mlx5_autoconf.h file is still generated.
 # input array for meson member search:
diff --git a/drivers/common/mlx5/linux/mlx5_common_auxiliary.c b/drivers/common/mlx5/linux/mlx5_common_auxiliary.c
new file mode 100644
index 0000000000..f16fd2ee37
--- /dev/null
+++ b/drivers/common/mlx5/linux/mlx5_common_auxiliary.c
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies Ltd
+ */
+
+#include <stdlib.h>
+#include <dirent.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include <rte_bus_auxiliary.h>
+#include <rte_common.h>
+#include "eal_filesystem.h"
+
+#include "mlx5_common_utils.h"
+#include "mlx5_common_private.h"
+
+#define AUXILIARY_SYSFS_PATH "/sys/bus/auxiliary/devices"
+#define MLX5_AUXILIARY_PREFIX "mlx5_core.sf."
+
+int
+mlx5_auxiliary_get_child_name(const char *dev, const char *node,
+			      char *child, size_t size)
+{
+	DIR *dir;
+	struct dirent *dent;
+	MKSTR(path, "%s/%s%s", AUXILIARY_SYSFS_PATH, dev, node);
+
+	dir = opendir(path);
+	if (dir == NULL) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	/* Get the first file name. */
+	while ((dent = readdir(dir)) != NULL) {
+		if (dent->d_name[0] != '.')
+			break;
+	}
+	closedir(dir);
+	if (dent == NULL) {
+		rte_errno = ENOENT;
+		return -rte_errno;
+	}
+	if (rte_strscpy(child, dent->d_name, size) < 0)
+		return -rte_errno;
+	return 0;
+}
+
+static int
+mlx5_auxiliary_get_pci_path(const struct rte_auxiliary_device *dev,
+			    char *sysfs_pci, size_t size)
+{
+	char sysfs_real[PATH_MAX];
+	char *last_slash;
+	MKSTR(sysfs_aux, "%s/%s", AUXILIARY_SYSFS_PATH, dev->name);
+
+	if (realpath(sysfs_aux, sysfs_real) == NULL) {
+		rte_errno = errno;
+		return -rte_errno;
+	}
+	last_slash = strrchr(sysfs_real, '/');
+	if (last_slash == NULL) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	*last_slash = '\0';
+	if (rte_strscpy(sysfs_pci, sysfs_real, size) < 0)
+		return -rte_errno;
+	return 0;
+}
+
+static int
+mlx5_auxiliary_get_numa(const struct rte_auxiliary_device *dev)
+{
+	unsigned long numa;
+	char numa_path[PATH_MAX];
+
+	if (mlx5_auxiliary_get_pci_path(dev, numa_path, sizeof(numa_path)) != 0)
+		return SOCKET_ID_ANY;
+	if (strcat(numa_path, "/numa_node") == NULL) {
+		rte_errno = ENAMETOOLONG;
+		return SOCKET_ID_ANY;
+	}
+	if (eal_parse_sysfs_value(numa_path, &numa) != 0) {
+		rte_errno = EINVAL;
+		return SOCKET_ID_ANY;
+	}
+	return (int)numa;
+}
+
+struct ibv_device *
+mlx5_get_aux_ibv_device(const struct rte_auxiliary_device *dev)
+{
+	int n;
+	char ib_name[64];
+	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
+	struct ibv_device *ibv_match = NULL;
+
+	if (!ibv_list) {
+		rte_errno = ENOSYS;
+		return NULL;
+	}
+	if (mlx5_auxiliary_get_child_name(dev->name, "/infiniband",
+					  ib_name, sizeof(ib_name)) != 0)
+		return NULL;
+	while (n-- > 0) {
+		if (strcmp(ibv_list[n]->name, ib_name) != 0)
+			continue;
+		ibv_match = ibv_list[n];
+		break;
+	}
+	if (ibv_match == NULL)
+		rte_errno = ENOENT;
+	mlx5_glue->free_device_list(ibv_list);
+	return ibv_match;
+}
+
+static bool
+mlx5_common_auxiliary_match(const char *name)
+{
+	return strncmp(name, MLX5_AUXILIARY_PREFIX,
+		       strlen(MLX5_AUXILIARY_PREFIX)) == 0;
+}
+
+static int
+mlx5_common_auxiliary_probe(struct rte_auxiliary_driver *drv __rte_unused,
+			    struct rte_auxiliary_device *dev)
+{
+	dev->device.numa_node = mlx5_auxiliary_get_numa(dev);
+	return mlx5_common_dev_probe(&dev->device);
+}
+
+static int
+mlx5_common_auxiliary_remove(struct rte_auxiliary_device *auxiliary_dev)
+{
+	return mlx5_common_dev_remove(&auxiliary_dev->device);
+}
+
+static int
+mlx5_common_auxiliary_dma_map(struct rte_auxiliary_device *auxiliary_dev,
+			      void *addr, uint64_t iova, size_t len)
+{
+	return mlx5_common_dev_dma_map(&auxiliary_dev->device, addr, iova, len);
+}
+
+static int
+mlx5_common_auxiliary_dma_unmap(struct rte_auxiliary_device *auxiliary_dev,
+				void *addr, uint64_t iova, size_t len)
+{
+	return mlx5_common_dev_dma_unmap(&auxiliary_dev->device, addr, iova,
+					 len);
+}
+
+static struct rte_auxiliary_driver mlx5_auxiliary_driver = {
+	.driver = {
+		   .name = MLX5_AUXILIARY_DRIVER_NAME,
+	},
+	.match = mlx5_common_auxiliary_match,
+	.probe = mlx5_common_auxiliary_probe,
+	.remove = mlx5_common_auxiliary_remove,
+	.dma_map = mlx5_common_auxiliary_dma_map,
+	.dma_unmap = mlx5_common_auxiliary_dma_unmap,
+};
+
+void mlx5_common_auxiliary_init(void)
+{
+	if (mlx5_auxiliary_driver.bus == NULL)
+		rte_auxiliary_register(&mlx5_auxiliary_driver);
+}
+
+RTE_FINI(mlx5_common_auxiliary_driver_finish)
+{
+	if (mlx5_auxiliary_driver.bus != NULL)
+		rte_auxiliary_unregister(&mlx5_auxiliary_driver);
+}
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
index a49440ef72..856e782878 100644
--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -12,6 +12,7 @@
 
 #include <rte_errno.h>
 #include <rte_bus_pci.h>
+#include <rte_bus_auxiliary.h>
 
 #include "mlx5_common_log.h"
 #include "mlx5_common_utils.h"
@@ -24,10 +25,12 @@
 struct ibv_device *
 mlx5_get_ibv_device(const struct rte_device *dev)
 {
-	struct ibv_device *ibv = NULL;
+	struct ibv_device *ibv;
 
 	if (mlx5_dev_is_pci(dev))
 		ibv = mlx5_get_pci_ibv_device(&RTE_DEV_TO_PCI_CONST(dev)->addr);
+	else
+		ibv = mlx5_get_aux_ibv_device(RTE_DEV_TO_AUXILIARY_CONST(dev));
 	if (ibv == NULL) {
 		rte_errno = ENODEV;
 		DRV_LOG(ERR, "Verbs device not found: %s", dev->name);
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index 875668d72b..b7be713cbe 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -356,6 +356,9 @@ mlx5_class_driver_register(struct mlx5_class_driver *driver)
 static void mlx5_common_driver_init(void)
 {
 	mlx5_common_pci_init();
+#ifdef RTE_EXEC_ENV_LINUX
+	mlx5_common_auxiliary_init();
+#endif
 }
 
 static bool mlx5_common_initialized;
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 62a0dc4bad..c9c77ce540 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -23,6 +23,7 @@
 
 /* Reported driver name. */
 #define MLX5_PCI_DRIVER_NAME "mlx5_pci"
+#define MLX5_AUXILIARY_DRIVER_NAME "mlx5_auxiliary"
 
 /* Bit-field manipulation. */
 #define BITFIELD_DECLARE(bf, type, size) \
@@ -140,6 +141,10 @@ struct ibv_device;
 __rte_internal
 struct ibv_device *mlx5_get_ibv_device(const struct rte_device *dev);
 
+__rte_internal
+int mlx5_auxiliary_get_child_name(const char *dev, const char *node,
+				  char *child, size_t size);
+
 /* Maximum number of simultaneous unicast MAC addresses. */
 #define MLX5_MAX_UC_MAC_ADDRESSES 128
 /* Maximum number of simultaneous Multicast MAC addresses. */
diff --git a/drivers/common/mlx5/mlx5_common_private.h b/drivers/common/mlx5/mlx5_common_private.h
index 1beeaae50e..d1ab15ac43 100644
--- a/drivers/common/mlx5/mlx5_common_private.h
+++ b/drivers/common/mlx5/mlx5_common_private.h
@@ -6,6 +6,7 @@
 #define _MLX5_COMMON_PRIVATE_H_
 
 #include <rte_pci.h>
+#include <rte_bus_auxiliary.h>
 
 #include "mlx5_common.h"
 
@@ -36,6 +37,10 @@ bool mlx5_dev_pci_match(const struct mlx5_class_driver *drv,
 			const struct rte_device *dev);
 struct ibv_device *mlx5_get_pci_ibv_device(const struct rte_pci_addr *);
 
+/* Common auxiliary bus driver: */
+void mlx5_common_auxiliary_init(void);
+struct ibv_device *mlx5_get_aux_ibv_device(const struct rte_auxiliary_device *);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index b10e1c4646..33eb7f09bc 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -3,6 +3,8 @@ INTERNAL {
 
 	haswell_broadwell_cpu;
 
+	mlx5_auxiliary_get_child_name;
+
 	mlx5_class_driver_register;
 
 	mlx5_common_init;
-- 
2.25.1