DPDK patches and discussions
 help / color / mirror / Atom feed
From: Xueming Li <xuemingl@nvidia.com>
To: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
Cc: <dev@dpdk.org>, <xuemingl@nvidia.com>,
	Matan Azrad <matan@nvidia.com>,
	Shahaf Shuler <shahafs@nvidia.com>, Ray Kinsella <mdr@ashroe.eu>
Subject: [dpdk-dev] [PATCH v3 02/15] common/mlx5: add common device driver
Date: Mon, 19 Jul 2021 10:53:57 +0800	[thread overview]
Message-ID: <20210719025410.15483-3-xuemingl@nvidia.com> (raw)
In-Reply-To: <20210713131437.30170-2-xuemingl@nvidia.com>

To support auxiliary bus, introduces common device driver and callbacks,
suppose to replace mlx5 common PCI bus driver.

Mlx5 class drivers, i.e. eth, vDPA, regex and compress normally consumes
single Verbs device context to probe a device. The Verbs device comes
from PCI address if the device is PCI bus device, from Auxiliary sysfs
if the device is auxiliary bus device. Currently only PCI bus is
supported.

Common device driver is a middle layer between mlx5 class drivers and
bus, resolve and abstract bus info to Verbs device for class drivers.
Both PCI bus driver and Auxiliary bus driver can utilize the common
driver layer to cast bus operations to mlx5 class drivers.

Legacy mlx5 common PCI bus driver still being used by mlx5 eth, vDPA,
regex and compress PMD, will remove once all PMD drivers migrate to new
common driver.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 doc/guides/rel_notes/release_21_08.rst        |   5 +
 drivers/common/mlx5/linux/mlx5_common_os.c    |   2 +-
 drivers/common/mlx5/linux/mlx5_common_os.h    |   7 +-
 drivers/common/mlx5/linux/mlx5_common_verbs.c |  21 +-
 drivers/common/mlx5/mlx5_common.c             | 362 +++++++++++++++++-
 drivers/common/mlx5/mlx5_common.h             | 128 +++++++
 drivers/common/mlx5/mlx5_common_pci.c         | 133 ++++++-
 drivers/common/mlx5/mlx5_common_private.h     |  41 ++
 drivers/common/mlx5/mlx5_common_utils.h       |   2 +
 drivers/common/mlx5/version.map               |   4 +
 10 files changed, 687 insertions(+), 18 deletions(-)
 create mode 100644 drivers/common/mlx5/mlx5_common_private.h

diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index 1b38b1aa51..768e5362f8 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -125,6 +125,11 @@ New Features
   The experimental PMD power management API now supports managing
   multiple Ethernet Rx queues per lcore.
 
+* **Added Sub-Function support for mlx5 PMDs**
+
+  Added Sub-Function support based on auxiliary bus for mlx5 PMDs:
+  net, vDPA, compress and regex.
+
 
 Removed Items
 -------------
diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index ea0b71e425..78a9723075 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -425,7 +425,7 @@ mlx5_glue_constructor(void)
 }
 
 struct ibv_device *
-mlx5_os_get_ibv_device(struct rte_pci_addr *addr)
+mlx5_os_get_ibv_device(const struct rte_pci_addr *addr)
 {
 	int n;
 	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
diff --git a/drivers/common/mlx5/linux/mlx5_common_os.h b/drivers/common/mlx5/linux/mlx5_common_os.h
index 72d6bf828b..86d0cb09b0 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.h
+++ b/drivers/common/mlx5/linux/mlx5_common_os.h
@@ -291,6 +291,11 @@ mlx5_os_free(void *addr)
 
 __rte_internal
 struct ibv_device *
-mlx5_os_get_ibv_device(struct rte_pci_addr *addr);
+mlx5_os_get_ibv_device(const struct rte_pci_addr *addr);
+
+__rte_internal
+struct ibv_device *
+mlx5_os_get_ibv_dev(const struct rte_device *dev);
+
 
 #endif /* RTE_PMD_MLX5_COMMON_OS_H_ */
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
index aa560f05f2..6a6ab7a7a2 100644
--- a/drivers/common/mlx5/linux/mlx5_common_verbs.c
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -10,11 +10,31 @@
 #include <sys/mman.h>
 #include <inttypes.h>
 
+#include <rte_errno.h>
+#include <rte_bus_pci.h>
+
+#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
+#include "mlx5_common_private.h"
 #include "mlx5_autoconf.h"
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
 #include <mlx5_common_mr.h>
 
+struct ibv_device *
+mlx5_os_get_ibv_dev(const struct rte_device *dev)
+{
+	struct ibv_device *ibv = NULL;
+
+	if (mlx5_dev_is_pci(dev))
+		ibv = mlx5_os_get_ibv_device(&RTE_DEV_TO_PCI_CONST(dev)->addr);
+	if (ibv == NULL) {
+		rte_errno = ENODEV;
+		DRV_LOG(ERR, "Verbs device not found: %s", dev->name);
+	}
+	return ibv;
+}
+
 /**
  * Register mr. Given protection domain pointer, pointer to addr and length
  * register the memory region.
@@ -68,4 +88,3 @@ mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
 		memset(pmd_mr, 0, sizeof(*pmd_mr));
 	}
 }
-
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index 25e9f09108..8734081a6e 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -8,11 +8,14 @@
 
 #include <rte_errno.h>
 #include <rte_mempool.h>
+#include <rte_class.h>
+#include <rte_malloc.h>
 
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
 #include "mlx5_common_log.h"
 #include "mlx5_common_pci.h"
+#include "mlx5_common_private.h"
 
 uint8_t haswell_broadwell_cpu;
 
@@ -41,6 +44,361 @@ static inline void mlx5_cpu_id(unsigned int level,
 
 RTE_LOG_REGISTER_DEFAULT(mlx5_common_logtype, NOTICE)
 
+/* Head of list of drivers. */
+static TAILQ_HEAD(mlx5_drivers, mlx5_class_driver) drivers_list =
+				TAILQ_HEAD_INITIALIZER(drivers_list);
+
+/* Head of devices. */
+static TAILQ_HEAD(mlx5_devices, mlx5_common_device) devices_list =
+				TAILQ_HEAD_INITIALIZER(devices_list);
+
+static const struct {
+	const char *name;
+	unsigned int drv_class;
+} mlx5_classes[] = {
+	{ .name = "vdpa", .drv_class = MLX5_CLASS_VDPA },
+	{ .name = "eth", .drv_class = MLX5_CLASS_ETH },
+	/* Keep class "net" for backward compatibility. */
+	{ .name = "net", .drv_class = MLX5_CLASS_ETH },
+	{ .name = "regex", .drv_class = MLX5_CLASS_REGEX },
+	{ .name = "compress", .drv_class = MLX5_CLASS_COMPRESS },
+};
+
+static int
+class_name_to_value(const char *class_name)
+{
+	unsigned int i;
+
+	for (i = 0; i < RTE_DIM(mlx5_classes); i++) {
+		if (strcmp(class_name, mlx5_classes[i].name) == 0)
+			return mlx5_classes[i].drv_class;
+	}
+	return -EINVAL;
+}
+
+static struct mlx5_class_driver *
+driver_get(uint32_t class)
+{
+	struct mlx5_class_driver *driver;
+
+	TAILQ_FOREACH(driver, &drivers_list, next) {
+		if ((uint32_t)driver->drv_class == class)
+			return driver;
+	}
+	return NULL;
+}
+
+static int
+devargs_class_handler(__rte_unused const char *key,
+		      const char *class_names, void *opaque)
+{
+	int *ret = opaque;
+	int class_val;
+	char *scratch;
+	char *found;
+	char *refstr = NULL;
+
+	*ret = 0;
+	scratch = strdup(class_names);
+	if (scratch == NULL) {
+		*ret = -ENOMEM;
+		return *ret;
+	}
+	found = strtok_r(scratch, ":", &refstr);
+	if (found == NULL)
+		/* Empty string. */
+		goto err;
+	do {
+		/* Extract each individual class name. Multiple
+		 * classes can be supplied as class=net:regex:foo:bar.
+		 */
+		class_val = class_name_to_value(found);
+		/* Check if its a valid class. */
+		if (class_val < 0) {
+			*ret = -EINVAL;
+			goto err;
+		}
+		*ret |= class_val;
+		found = strtok_r(NULL, ":", &refstr);
+	} while (found != NULL);
+err:
+	free(scratch);
+	if (*ret < 0)
+		DRV_LOG(ERR, "Invalid mlx5 class options: %s.\n", class_names);
+	return *ret;
+}
+
+static int
+parse_class_options(const struct rte_devargs *devargs)
+{
+	struct rte_kvargs *kvlist;
+	int ret = 0;
+
+	if (devargs == NULL)
+		return 0;
+	if (devargs->cls != NULL && devargs->cls->name != NULL)
+		/* Global syntax, only one class type. */
+		return class_name_to_value(devargs->cls->name);
+	/* Legacy devargs support multiple classes. */
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL)
+		return 0;
+	rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_CLASS,
+			   devargs_class_handler, &ret);
+	rte_kvargs_free(kvlist);
+	return ret;
+}
+
+static const unsigned int mlx5_class_invalid_combinations[] = {
+	MLX5_CLASS_ETH | MLX5_CLASS_VDPA,
+	/* New class combination should be added here. */
+};
+
+static int
+is_valid_class_combination(uint32_t user_classes)
+{
+	unsigned int i;
+
+	/* Verify if user specified unsupported combination. */
+	for (i = 0; i < RTE_DIM(mlx5_class_invalid_combinations); i++) {
+		if ((mlx5_class_invalid_combinations[i] & user_classes) ==
+		    mlx5_class_invalid_combinations[i])
+			return -EINVAL;
+	}
+	/* Not found any invalid class combination. */
+	return 0;
+}
+
+static bool
+device_class_enabled(const struct mlx5_common_device *device, uint32_t class)
+{
+	return (device->classes_loaded & class) > 0;
+}
+
+static bool
+mlx5_bus_match(const struct mlx5_class_driver *drv,
+	       const struct rte_device *dev)
+{
+	if (mlx5_dev_is_pci(dev))
+		return mlx5_dev_pci_match(drv, dev);
+	return true;
+}
+
+static struct mlx5_common_device *
+to_mlx5_device(const struct rte_device *rte_dev)
+{
+	struct mlx5_common_device *dev;
+
+	TAILQ_FOREACH(dev, &devices_list, next) {
+		if (rte_dev == dev->dev)
+			return dev;
+	}
+	return NULL;
+}
+
+static void
+dev_release(struct mlx5_common_device *dev)
+{
+	TAILQ_REMOVE(&devices_list, dev, next);
+	rte_free(dev);
+}
+
+static int
+drivers_remove(struct mlx5_common_device *dev, uint32_t enabled_classes)
+{
+	struct mlx5_class_driver *driver;
+	int local_ret = -ENODEV;
+	unsigned int i = 0;
+	int ret = 0;
+
+	enabled_classes &= dev->classes_loaded;
+	while (enabled_classes) {
+		driver = driver_get(RTE_BIT64(i));
+		if (driver != NULL) {
+			local_ret = driver->remove(dev->dev);
+			if (local_ret == 0)
+				dev->classes_loaded &= ~RTE_BIT64(i);
+			else if (ret == 0)
+				ret = local_ret;
+		}
+		enabled_classes &= ~RTE_BIT64(i);
+		i++;
+	}
+	if (local_ret != 0 && ret == 0)
+		ret = local_ret;
+	return ret;
+}
+
+static int
+drivers_probe(struct mlx5_common_device *dev, uint32_t user_classes)
+{
+	struct mlx5_class_driver *driver;
+	uint32_t enabled_classes = 0;
+	bool already_loaded;
+	int ret;
+
+	TAILQ_FOREACH(driver, &drivers_list, next) {
+		if ((driver->drv_class & user_classes) == 0)
+			continue;
+		if (!mlx5_bus_match(driver, dev->dev))
+			continue;
+		already_loaded = dev->classes_loaded & driver->drv_class;
+		if (already_loaded && driver->probe_again == 0) {
+			DRV_LOG(ERR, "Device %s is already probed",
+				dev->dev->name);
+			ret = -EEXIST;
+			goto probe_err;
+		}
+		ret = driver->probe(dev->dev);
+		if (ret < 0) {
+			DRV_LOG(ERR, "Failed to load driver %s",
+				driver->name);
+			goto probe_err;
+		}
+		enabled_classes |= driver->drv_class;
+	}
+	dev->classes_loaded |= enabled_classes;
+	return 0;
+probe_err:
+	/* Only unload drivers which are enabled which were enabled
+	 * in this probe instance.
+	 */
+	drivers_remove(dev, enabled_classes);
+	return ret;
+}
+
+int
+mlx5_common_dev_probe(struct rte_device *eal_dev)
+{
+	struct mlx5_common_device *dev;
+	uint32_t classes = 0;
+	bool new_device = false;
+	int ret;
+
+	DRV_LOG(INFO, "probe device \"%s\".", eal_dev->name);
+	ret = parse_class_options(eal_dev->devargs);
+	if (ret < 0) {
+		DRV_LOG(ERR, "Unsupported mlx5 class type: %s",
+			eal_dev->devargs->args);
+		return ret;
+	}
+	classes = ret;
+	if (classes == 0)
+		/* Default to net class. */
+		classes = MLX5_CLASS_ETH;
+	dev = to_mlx5_device(eal_dev);
+	if (!dev) {
+		dev = rte_zmalloc("mlx5_common_device", sizeof(*dev), 0);
+		if (!dev)
+			return -ENOMEM;
+		dev->dev = eal_dev;
+		TAILQ_INSERT_HEAD(&devices_list, dev, next);
+		new_device = true;
+	} else {
+		/* Validate combination here. */
+		ret = is_valid_class_combination(classes |
+						 dev->classes_loaded);
+		if (ret != 0) {
+			DRV_LOG(ERR, "Unsupported mlx5 classes combination.");
+			return ret;
+		}
+	}
+	ret = drivers_probe(dev, classes);
+	if (ret)
+		goto class_err;
+	return 0;
+class_err:
+	if (new_device)
+		dev_release(dev);
+	return ret;
+}
+
+int
+mlx5_common_dev_remove(struct rte_device *eal_dev)
+{
+	struct mlx5_common_device *dev;
+	int ret;
+
+	dev = to_mlx5_device(eal_dev);
+	if (!dev)
+		return -ENODEV;
+	/* Matching device found, cleanup and unload drivers. */
+	ret = drivers_remove(dev, dev->classes_loaded);
+	if (ret != 0)
+		dev_release(dev);
+	return ret;
+}
+
+int
+mlx5_common_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova,
+			size_t len)
+{
+	struct mlx5_class_driver *driver = NULL;
+	struct mlx5_class_driver *temp;
+	struct mlx5_common_device *mdev;
+	int ret = -EINVAL;
+
+	mdev = to_mlx5_device(dev);
+	if (!mdev)
+		return -ENODEV;
+	TAILQ_FOREACH(driver, &drivers_list, next) {
+		if (!device_class_enabled(mdev, driver->drv_class) ||
+		    driver->dma_map == NULL)
+			continue;
+		ret = driver->dma_map(dev, addr, iova, len);
+		if (ret)
+			goto map_err;
+	}
+	return ret;
+map_err:
+	TAILQ_FOREACH(temp, &drivers_list, next) {
+		if (temp == driver)
+			break;
+		if (device_class_enabled(mdev, temp->drv_class) &&
+		    temp->dma_map && temp->dma_unmap)
+			temp->dma_unmap(dev, addr, iova, len);
+	}
+	return ret;
+}
+
+int
+mlx5_common_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+			  size_t len)
+{
+	struct mlx5_class_driver *driver;
+	struct mlx5_common_device *mdev;
+	int local_ret = -EINVAL;
+	int ret = 0;
+
+	mdev = to_mlx5_device(dev);
+	if (!mdev)
+		return -ENODEV;
+	/* There is no unmap error recovery in current implementation. */
+	TAILQ_FOREACH_REVERSE(driver, &drivers_list, mlx5_drivers, next) {
+		if (!device_class_enabled(mdev, driver->drv_class) ||
+		    driver->dma_unmap == NULL)
+			continue;
+		local_ret = driver->dma_unmap(dev, addr, iova, len);
+		if (local_ret && (ret == 0))
+			ret = local_ret;
+	}
+	if (local_ret)
+		ret = local_ret;
+	return ret;
+}
+
+void
+mlx5_class_driver_register(struct mlx5_class_driver *driver)
+{
+	mlx5_common_driver_on_register_pci(driver);
+	TAILQ_INSERT_TAIL(&drivers_list, driver, next);
+}
+
+static void mlx5_common_driver_init(void)
+{
+	mlx5_common_pci_init();
+}
+
 static bool mlx5_common_initialized;
 
 /**
@@ -55,7 +413,7 @@ mlx5_common_init(void)
 		return;
 
 	mlx5_glue_constructor();
-	mlx5_common_pci_init();
+	mlx5_common_driver_init();
 	mlx5_common_initialized = true;
 }
 
@@ -214,3 +572,5 @@ mlx5_devx_alloc_uar(void *ctx, int mapping)
 exit:
 	return uar;
 }
+
+RTE_PMD_EXPORT_NAME(mlx5_common_driver, __COUNTER__);
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index e8cd5182ac..a2a69ebf5e 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -244,4 +244,132 @@ extern uint8_t haswell_broadwell_cpu;
 __rte_internal
 void mlx5_common_init(void);
 
+/**
+ * Common Driver Interface
+ * ConnectX common driver supports multiple classes: net,vdpa,regex,crypto and
+ * compress devices. This layer enables creating such multiple class of devices
+ * on a single device by allowing to bind multiple class specific device
+ * driver to attach to common driver.
+ *
+ * ------------  -------------  --------------  -----------------  ------------
+ * | mlx5 net |  | mlx5 vdpa |  | mlx5 regex |  | mlx5 compress |  | mlx5 ... |
+ * |  driver  |  |  driver   |  |   driver   |  |     driver    |  |  drivers |
+ * ------------  -------------  --------------  -----------------  ------------
+ *                               ||
+ *                        -----------------
+ *                        |     mlx5      |
+ *                        | common driver |
+ *                        -----------------
+ *                          |          |
+ *                 -----------        -----------------
+ *                 |   mlx5  |        |   mlx5        |
+ *                 | pci dev |        | auxiliary dev |
+ *                 -----------        -----------------
+ *
+ * - mlx5 pci bus driver binds to mlx5 PCI devices defined by PCI
+ *   ID table of all related mlx5 PCI devices.
+ * - mlx5 class driver such as net, vdpa, regex PMD defines its
+ *   specific PCI ID table and mlx5 bus driver probes matching
+ *   class drivers.
+ * - mlx5 common driver is central place that validates supported
+ *   class combinations.
+ * - mlx5 common driver hide bus difference by resolving device address
+ *   from devargs, locating target RDMA device and probing with it.
+ */
+
+/**
+ * Initialization function for the driver called during device probing.
+ */
+typedef int (mlx5_class_driver_probe_t)(struct rte_device *dev);
+
+/**
+ * Uninitialization function for the driver called during hot-unplugging.
+ */
+typedef int (mlx5_class_driver_remove_t)(struct rte_device *dev);
+
+/**
+ * Driver-specific DMA mapping. After a successful call the device
+ * will be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (mlx5_class_driver_dma_map_t)(struct rte_device *dev, void *addr,
+					  uint64_t iova, size_t len);
+
+/**
+ * Driver-specific DMA un-mapping. After a successful call the device
+ * will not be able to read/write from/to this segment.
+ *
+ * @param dev
+ *   Pointer to the device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ * @return
+ *   - 0 On success.
+ *   - Negative value and rte_errno is set otherwise.
+ */
+typedef int (mlx5_class_driver_dma_unmap_t)(struct rte_device *dev, void *addr,
+					    uint64_t iova, size_t len);
+
+/** Device already probed can be probed again to check for new ports. */
+#define MLX5_DRV_PROBE_AGAIN 0x0004
+
+/**
+ * A structure describing a mlx5 common class driver.
+ */
+struct mlx5_class_driver {
+	TAILQ_ENTRY(mlx5_class_driver) next;
+	enum mlx5_class drv_class;            /**< Class of this driver. */
+	const char *name;                     /**< Driver name. */
+	mlx5_class_driver_probe_t *probe;     /**< Device Probe function. */
+	mlx5_class_driver_remove_t *remove;   /**< Device Remove function. */
+	mlx5_class_driver_dma_map_t *dma_map; /**< device dma map function. */
+	mlx5_class_driver_dma_unmap_t *dma_unmap;
+	/**< device dma unmap function. */
+	const struct rte_pci_id *id_table;    /**< ID table, NULL terminated. */
+	uint32_t probe_again:1;
+	/**< Device already probed can be probed again to check new device. */
+	uint32_t intr_lsc:1; /**< Supports link state interrupt. */
+	uint32_t intr_rmv:1; /**< Supports device remove interrupt. */
+};
+
+/**
+ * Register a mlx5 device driver.
+ *
+ * @param driver
+ *   A pointer to a mlx5_driver structure describing the driver
+ *   to be registered.
+ */
+__rte_internal
+void
+mlx5_class_driver_register(struct mlx5_class_driver *driver);
+
+/**
+ * Test a device is PCI bus device.
+ *
+ * @param dev
+ *   Pointer to device.
+ *
+ * @return
+ *   - True on device devargs is a PCI bus device.
+ *   - False otherwise.
+ */
+__rte_internal
+bool
+mlx5_dev_is_pci(const struct rte_device *dev);
+
 #endif /* RTE_PMD_MLX5_COMMON_H_ */
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 1e3c1de12d..f2d2015f9d 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -3,11 +3,19 @@
  */
 
 #include <stdlib.h>
+
 #include <rte_malloc.h>
+#include <rte_devargs.h>
+#include <rte_errno.h>
 #include <rte_class.h>
 
 #include "mlx5_common_log.h"
 #include "mlx5_common_pci.h"
+#include "mlx5_common_private.h"
+
+static struct rte_pci_driver mlx5_common_pci_driver;
+
+/********** Legacy PCI bus driver, to be removed ********/
 
 struct mlx5_pci_device {
 	struct rte_pci_device *pci_dev;
@@ -284,8 +292,8 @@ drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv,
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
-		      struct rte_pci_device *pci_dev)
+mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	       struct rte_pci_device *pci_dev)
 {
 	struct mlx5_pci_device *dev;
 	uint32_t user_classes = 0;
@@ -338,7 +346,7 @@ mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
  *   0 on success, the function cannot fail.
  */
 static int
-mlx5_common_pci_remove(struct rte_pci_device *pci_dev)
+mlx5_pci_remove(struct rte_pci_device *pci_dev)
 {
 	struct mlx5_pci_device *dev;
 	int ret;
@@ -354,8 +362,8 @@ mlx5_common_pci_remove(struct rte_pci_device *pci_dev)
 }
 
 static int
-mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
-			uint64_t iova, size_t len)
+mlx5_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
+		 uint64_t iova, size_t len)
 {
 	struct mlx5_pci_driver *driver = NULL;
 	struct mlx5_pci_driver *temp;
@@ -387,8 +395,8 @@ mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
 }
 
 static int
-mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
-			  uint64_t iova, size_t len)
+mlx5_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
+		   uint64_t iova, size_t len)
 {
 	struct mlx5_pci_driver *driver;
 	struct mlx5_pci_device *dev;
@@ -421,10 +429,10 @@ static struct rte_pci_driver mlx5_pci_driver = {
 	.driver = {
 		.name = MLX5_PCI_DRIVER_NAME,
 	},
-	.probe = mlx5_common_pci_probe,
-	.remove = mlx5_common_pci_remove,
-	.dma_map = mlx5_common_pci_dma_map,
-	.dma_unmap = mlx5_common_pci_dma_unmap,
+	.probe = mlx5_pci_probe,
+	.remove = mlx5_pci_remove,
+	.dma_map = mlx5_pci_dma_map,
+	.dma_unmap = mlx5_pci_dma_unmap,
 };
 
 static int
@@ -488,7 +496,7 @@ pci_ids_table_update(const struct rte_pci_id *driver_id_table)
 	updated_table = calloc(num_ids, sizeof(*updated_table));
 	if (!updated_table)
 		return -ENOMEM;
-	if (TAILQ_EMPTY(&drv_list)) {
+	if (old_table == NULL) {
 		/* Copy the first driver's ID table. */
 		for (id_iter = driver_id_table; id_iter->vendor_id != 0;
 		     id_iter++, i++)
@@ -504,6 +512,7 @@ pci_ids_table_update(const struct rte_pci_id *driver_id_table)
 	/* Terminate table with empty entry. */
 	updated_table[i].vendor_id = 0;
 	mlx5_pci_driver.id_table = updated_table;
+	mlx5_common_pci_driver.id_table = updated_table;
 	mlx5_pci_id_table = updated_table;
 	if (old_table)
 		free(old_table);
@@ -522,6 +531,101 @@ mlx5_pci_driver_register(struct mlx5_pci_driver *driver)
 	TAILQ_INSERT_TAIL(&drv_list, driver, next);
 }
 
+/********** New common PCI bus driver ********/
+
+bool
+mlx5_dev_is_pci(const struct rte_device *dev)
+{
+	return strcmp(dev->bus->name, "pci") == 0;
+}
+
+bool
+mlx5_dev_pci_match(const struct mlx5_class_driver *drv,
+		   const struct rte_device *dev)
+{
+	const struct rte_pci_device *pci_dev;
+	const struct rte_pci_id *id_table;
+
+	if (!mlx5_dev_is_pci(dev))
+		return false;
+	pci_dev = RTE_DEV_TO_PCI_CONST(dev);
+	for (id_table = drv->id_table; id_table->vendor_id != 0;
+	     id_table++) {
+		/* Check if device's ids match the class driver's ids. */
+		if (id_table->vendor_id != pci_dev->id.vendor_id &&
+		    id_table->vendor_id != RTE_PCI_ANY_ID)
+			continue;
+		if (id_table->device_id != pci_dev->id.device_id &&
+		    id_table->device_id != RTE_PCI_ANY_ID)
+			continue;
+		if (id_table->subsystem_vendor_id !=
+		    pci_dev->id.subsystem_vendor_id &&
+		    id_table->subsystem_vendor_id != RTE_PCI_ANY_ID)
+			continue;
+		if (id_table->subsystem_device_id !=
+		    pci_dev->id.subsystem_device_id &&
+		    id_table->subsystem_device_id != RTE_PCI_ANY_ID)
+			continue;
+		if (id_table->class_id != pci_dev->id.class_id &&
+		    id_table->class_id != RTE_CLASS_ANY_ID)
+			continue;
+		return true;
+	}
+	return false;
+}
+
+static int
+mlx5_common_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+		      struct rte_pci_device *pci_dev)
+{
+	return mlx5_common_dev_probe(&pci_dev->device);
+}
+
+static int
+mlx5_common_pci_remove(struct rte_pci_device *pci_dev)
+{
+	return mlx5_common_dev_remove(&pci_dev->device);
+}
+
+static int
+mlx5_common_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
+			uint64_t iova, size_t len)
+{
+	return mlx5_common_dev_dma_map(&pci_dev->device, addr, iova, len);
+}
+
+static int
+mlx5_common_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
+			  uint64_t iova, size_t len)
+{
+	return mlx5_common_dev_dma_unmap(&pci_dev->device, addr, iova, len);
+}
+
+void
+mlx5_common_driver_on_register_pci(struct mlx5_class_driver *driver)
+{
+	if (driver->id_table != NULL) {
+		if (pci_ids_table_update(driver->id_table) != 0)
+			return;
+	}
+	if (driver->probe_again)
+		mlx5_common_pci_driver.drv_flags |= RTE_PCI_DRV_PROBE_AGAIN;
+	if (driver->intr_lsc)
+		mlx5_common_pci_driver.drv_flags |= RTE_PCI_DRV_INTR_LSC;
+	if (driver->intr_rmv)
+		mlx5_common_pci_driver.drv_flags |= RTE_PCI_DRV_INTR_RMV;
+}
+
+static struct rte_pci_driver mlx5_common_pci_driver = {
+	.driver = {
+		   .name = MLX5_PCI_DRIVER_NAME,
+	},
+	.probe = mlx5_common_pci_probe,
+	.remove = mlx5_common_pci_remove,
+	.dma_map = mlx5_common_pci_dma_map,
+	.dma_unmap = mlx5_common_pci_dma_unmap,
+};
+
 void mlx5_common_pci_init(void)
 {
 	const struct rte_pci_id empty_table[] = {
@@ -537,7 +641,7 @@ void mlx5_common_pci_init(void)
 	 */
 	if (mlx5_pci_id_table == NULL && pci_ids_table_update(empty_table))
 		return;
-	rte_pci_register(&mlx5_pci_driver);
+	rte_pci_register(&mlx5_common_pci_driver);
 }
 
 RTE_FINI(mlx5_common_pci_finish)
@@ -546,8 +650,9 @@ RTE_FINI(mlx5_common_pci_finish)
 		/* Constructor doesn't register with PCI bus if it failed
 		 * to build the table.
 		 */
-		rte_pci_unregister(&mlx5_pci_driver);
+		rte_pci_unregister(&mlx5_common_pci_driver);
 		free(mlx5_pci_id_table);
 	}
 }
+
 RTE_PMD_EXPORT_NAME(mlx5_common_pci, __COUNTER__);
diff --git a/drivers/common/mlx5/mlx5_common_private.h b/drivers/common/mlx5/mlx5_common_private.h
new file mode 100644
index 0000000000..791eb3cd77
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_private.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef _MLX5_COMMON_PRIVATE_H_
+#define _MLX5_COMMON_PRIVATE_H_
+
+#include <rte_pci.h>
+
+#include "mlx5_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+/* Common bus driver: */
+
+struct mlx5_common_device {
+	struct rte_device *dev;
+	TAILQ_ENTRY(mlx5_common_device) next;
+	uint32_t classes_loaded;
+};
+
+int mlx5_common_dev_probe(struct rte_device *eal_dev);
+int mlx5_common_dev_remove(struct rte_device *eal_dev);
+int mlx5_common_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova,
+			    size_t len);
+int mlx5_common_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
+			      size_t len);
+
+/* Common PCI bus driver: */
+
+void mlx5_common_driver_on_register_pci(struct mlx5_class_driver *driver);
+bool mlx5_dev_pci_match(const struct mlx5_class_driver *drv,
+			const struct rte_device *dev);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* _MLX5_COMMON_PRIVATE_H_ */
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 613d29de0c..98e487e7ef 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -5,6 +5,8 @@
 #ifndef RTE_PMD_MLX5_COMMON_UTILS_H_
 #define RTE_PMD_MLX5_COMMON_UTILS_H_
 
+#include <rte_rwlock.h>
+
 #include "mlx5_common.h"
 
 /************************ mlx5 list *****************************/
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index e6586d6f6f..e9d43dc1e5 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -3,6 +3,8 @@ INTERNAL {
 
 	haswell_broadwell_cpu;
 
+	mlx5_class_driver_register;
+
 	mlx5_common_init;
 
 	mlx5_common_verbs_reg_mr; # WINDOWS_NO_EXPORT
@@ -10,6 +12,7 @@ INTERNAL {
 
 	mlx5_create_mr_ext;
 
+	mlx5_dev_is_pci; # WINDOWS_NO_EXPORT
 	mlx5_dev_to_pci_addr; # WINDOWS_NO_EXPORT
 
 	mlx5_devx_alloc_uar; # WINDOWS_NO_EXPORT
@@ -137,6 +140,7 @@ INTERNAL {
 	mlx5_os_alloc_pd;
 	mlx5_os_dealloc_pd;
 	mlx5_os_dereg_mr;
+	mlx5_os_get_ibv_dev; # WINDOWS_NO_EXPORT
 	mlx5_os_get_ibv_device; # WINDOWS_NO_EXPORT
 	mlx5_os_reg_mr;
 	mlx5_os_umem_dereg;
-- 
2.25.1


  parent reply	other threads:[~2021-07-19  2:55 UTC|newest]

Thread overview: 108+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-05-27 13:37 [dpdk-dev] [RFC 00/14] mlx5: support SubFunction Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 01/14] common/mlx5: add common device driver Xueming Li
2021-06-10  9:51   ` Thomas Monjalon
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 00/14] net/mlx5: support Sub-Function Xueming Li
2021-07-21 14:37     ` [dpdk-dev] [PATCH v4 00/16] " Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 01/16] common/mlx5: rename eth device class name Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 02/16] common/mlx5: add common device driver Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 03/16] common/mlx5: move description of PCI sysfs functions Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 04/16] common/mlx5: support auxiliary bus Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 05/16] common/mlx5: get PCI device address from any bus Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 06/16] net/mlx5: remove PCI dependency Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 07/16] net/mlx5: migrate to bus-agnostic common driver Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 08/16] net/mlx5: support SubFunction Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 09/16] net/mlx5: check max Verbs port number Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 10/16] regex/mlx5: migrate to common driver Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 11/16] vdpa/mlx5: define driver name as macro Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 12/16] vdpa/mlx5: remove PCI specifics Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 13/16] vdpa/mlx5: support SubFunction Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 14/16] compress/mlx5: migrate to common driver Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 15/16] crypto/mlx5: " Xueming Li
2021-07-21 14:37       ` [dpdk-dev] [PATCH v4 16/16] common/mlx5: clean up legacy PCI bus driver Xueming Li
2021-07-21 22:24       ` [dpdk-dev] [PATCH v4 00/16] net/mlx5: support Sub-Function Thomas Monjalon
2021-07-22  3:03         ` Xueming(Steven) Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 01/14] common/mlx5: add common device driver Xueming Li
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 00/14] net/mlx5: support Sub-Function Xueming Li
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 01/14] common/mlx5: add common device driver Xueming Li
2021-07-14  5:58       ` Slava Ovsiienko
2021-07-18 18:28       ` Thomas Monjalon
2021-07-19  4:05         ` Xueming(Steven) Li
2021-07-19  2:53       ` [dpdk-dev] [PATCH v3 00/15] net/mlx5: support Sub-Function Xueming Li
2021-07-19  2:53       ` [dpdk-dev] [PATCH v3 01/15] common/mlx5: rename eth device class name Xueming Li
2021-07-19  2:53       ` Xueming Li [this message]
2021-07-19  2:53       ` [dpdk-dev] [PATCH v3 03/15] common/mlx5: move description of PCI sysfs functions Xueming Li
2021-07-19  2:53       ` [dpdk-dev] [PATCH v3 04/15] common/mlx5: support auxiliary bus Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 05/15] common/mlx5: get PCI device address from any bus Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 06/15] net/mlx5: remove PCI dependency Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 07/15] net/mlx5: migrate to bus-agnostic common driver Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 08/15] net/mlx5: support SubFunction Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 09/15] net/mlx5: check max Verbs port number Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 10/15] regex/mlx5: migrate to common driver Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 11/15] vdpa/mlx5: define driver name as macro Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 12/15] vdpa/mlx5: remove PCI specifics Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 13/15] vdpa/mlx5: support SubFunction Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 14/15] compress/mlx5: migrate to common driver Xueming Li
2021-07-19  2:54       ` [dpdk-dev] [PATCH v3 15/15] common/mlx5: clean up legacy PCI bus driver Xueming Li
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 02/14] common/mlx5: move description of PCI sysfs functions Xueming Li
2021-07-14  5:58       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 03/14] common/mlx5: support auxiliary bus Xueming Li
2021-07-14  5:58       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 04/14] common/mlx5: get PCI device address from any bus Xueming Li
2021-07-14  5:59       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 05/14] net/mlx5: remove PCI dependency Xueming Li
2021-07-14  5:59       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 06/14] net/mlx5: migrate to bus-agnostic common driver Xueming Li
2021-07-14  5:59       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 07/14] net/mlx5: support SubFunction Xueming Li
2021-07-14  5:59       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 08/14] net/mlx5: check max Verbs port number Xueming Li
2021-07-14  6:00       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 09/14] regex/mlx5: migrate to common driver Xueming Li
2021-07-14  6:00       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 10/14] vdpa/mlx5: define driver name as macro Xueming Li
2021-07-14  6:00       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 11/14] vdpa/mlx5: remove PCI specifics Xueming Li
2021-07-14  6:08       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 12/14] vdpa/mlx5: support SubFunction Xueming Li
2021-07-14  6:01       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 13/14] compress/mlx5: migrate to common driver Xueming Li
2021-07-14  6:01       ` Slava Ovsiienko
2021-07-13 13:14     ` [dpdk-dev] [PATCH v2 14/14] common/mlx5: clean up legacy PCI bus driver Xueming Li
2021-07-14  6:01       ` Slava Ovsiienko
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 02/14] common/mlx5: move description of PCI sysfs functions Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 03/14] common/mlx5: support auxiliary bus Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 04/14] common/mlx5: get PCI device address from any bus Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 05/14] net/mlx5: remove PCI dependency Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 06/14] net/mlx5: migrate to bus-agnostic common driver Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 07/14] net/mlx5: support SubFunction Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 08/14] net/mlx5: check max Verbs port number Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 09/14] regex/mlx5: migrate to common driver Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 10/14] vdpa/mlx5: define driver name as macro Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 11/14] vdpa/mlx5: remove PCI specifics Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 12/14] vdpa/mlx5: support SubFunction Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 13/14] compress/mlx5: migrate to common driver Xueming Li
2021-06-16  4:09   ` [dpdk-dev] [PATCH v1 14/14] common/mlx5: clean up legacy PCI bus driver Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 02/14] common/mlx5: move description of PCI sysfs functions Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 03/14] net/mlx5: remove PCI dependency Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 04/14] net/mlx5: migrate to bus-agnostic common driver Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 05/14] regex/mlx5: migrate to " Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 06/14] compress/mlx5: " Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 07/14] vdpa/mlx5: fix driver name Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 08/14] vdpa/mlx5: remove PCI specifics Xueming Li
2021-05-27 13:37 ` [dpdk-dev] [RFC 09/14] common/mlx5: clean up legacy PCI bus driver Xueming Li
2021-05-27 14:01 ` [dpdk-dev] [RFC 10/14] bus/auxiliary: introduce auxiliary bus Xueming Li
2021-05-27 14:01   ` [dpdk-dev] [RFC 11/14] common/mlx5: support " Xueming Li
2021-05-27 14:02   ` [dpdk-dev] [RFC 12/14] common/mlx5: get PCI device address from any bus Xueming Li
2021-05-27 14:02   ` [dpdk-dev] [RFC 13/14] vdpa/mlx5: support SubFunction Xueming Li
2021-05-27 14:02   ` [dpdk-dev] [RFC 14/14] net/mlx5: " Xueming Li
2021-06-10 10:33 ` [dpdk-dev] [RFC 00/14] mlx5: " Ferruh Yigit
2021-06-10 13:23   ` Thomas Monjalon
2021-06-11  5:14     ` Xia, Chenbo
2021-06-11  7:54       ` Thomas Monjalon
2021-06-15  2:10         ` Xia, Chenbo
2021-06-15  4:04           ` Parav Pandit
2021-06-15  5:33             ` Xia, Chenbo
2021-06-15  5:43               ` Parav Pandit
2021-06-15 11:19                 ` Xia, Chenbo
2021-06-15 12:47                   ` Parav Pandit
2021-06-15 15:19                     ` Jason Gunthorpe

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210719025410.15483-3-xuemingl@nvidia.com \
    --to=xuemingl@nvidia.com \
    --cc=dev@dpdk.org \
    --cc=matan@nvidia.com \
    --cc=mdr@ashroe.eu \
    --cc=shahafs@nvidia.com \
    --cc=viacheslavo@nvidia.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).