[dpdk-dev] [PATCH v3 15/15] common/mlx5: clean up legacy PCI bus driver

[Xueming Li <xuemingl@nvidia.com>]

Clean up legacy PCI bus driver since all mlx5 PMDs moved to new common
PCI bus driver.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.h |   1 -
 drivers/common/mlx5/mlx5_common.c          |   1 -
 drivers/common/mlx5/mlx5_common.h          |   1 +
 drivers/common/mlx5/mlx5_common_pci.c      | 435 +--------------------
 drivers/common/mlx5/mlx5_common_pci.h      |  77 ----
 drivers/common/mlx5/mlx5_common_private.h  |   1 +
 drivers/common/mlx5/version.map            |   3 -
 7 files changed, 4 insertions(+), 515 deletions(-)
 delete mode 100644 drivers/common/mlx5/mlx5_common_pci.h

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.h b/drivers/common/mlx5/linux/mlx5_common_os.h
index 86d0cb09b0..2b03bf811e 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.h
+++ b/drivers/common/mlx5/linux/mlx5_common_os.h
@@ -289,7 +289,6 @@ mlx5_os_free(void *addr)
 	free(addr);
 }
 
-__rte_internal
 struct ibv_device *
 mlx5_os_get_ibv_device(const struct rte_pci_addr *addr);
 
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index c258cd127a..558474c706 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -14,7 +14,6 @@
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
 #include "mlx5_common_log.h"
-#include "mlx5_common_pci.h"
 #include "mlx5_common_private.h"
 
 uint8_t haswell_broadwell_cpu;
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 26d1b58853..f28ca938cf 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -10,6 +10,7 @@
 #include <rte_pci.h>
 #include <rte_debug.h>
 #include <rte_atomic.h>
+#include <rte_rwlock.h>
 #include <rte_log.h>
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index f2d2015f9d..8b38091d87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -8,433 +8,17 @@
 #include <rte_devargs.h>
 #include <rte_errno.h>
 #include <rte_class.h>
+#include <rte_pci.h>
+#include <rte_bus_pci.h>
 
 #include "mlx5_common_log.h"
-#include "mlx5_common_pci.h"
 #include "mlx5_common_private.h"
 
 static struct rte_pci_driver mlx5_common_pci_driver;
 
-/********** Legacy PCI bus driver, to be removed ********/
-
-struct mlx5_pci_device {
-	struct rte_pci_device *pci_dev;
-	TAILQ_ENTRY(mlx5_pci_device) next;
-	uint32_t classes_loaded;
-};
-
-/* Head of list of drivers. */
-static TAILQ_HEAD(mlx5_pci_bus_drv_head, mlx5_pci_driver) drv_list =
-				TAILQ_HEAD_INITIALIZER(drv_list);
-
-/* Head of mlx5 pci devices. */
-static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
-				TAILQ_HEAD_INITIALIZER(devices_list);
-
-static const struct {
-	const char *name;
-	unsigned int driver_class;
-} mlx5_classes[] = {
-	{ .name = "vdpa", .driver_class = MLX5_CLASS_VDPA },
-	{ .name = "eth", .driver_class = MLX5_CLASS_ETH },
-	/* Keep name "net" for backward compatibility. */
-	{ .name = "net", .driver_class = MLX5_CLASS_ETH },
-	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
-	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
-};
-
-static const unsigned int mlx5_class_combinations[] = {
-	MLX5_CLASS_ETH,
-	MLX5_CLASS_VDPA,
-	MLX5_CLASS_REGEX,
-	MLX5_CLASS_COMPRESS,
-	MLX5_CLASS_ETH | MLX5_CLASS_REGEX,
-	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
-	MLX5_CLASS_ETH | MLX5_CLASS_COMPRESS,
-	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
-	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
-	MLX5_CLASS_ETH | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
-	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
-	/* New class combination should be added here. */
-};
-
-static int
-class_name_to_value(const char *class_name)
-{
-	unsigned int i;
-
-	for (i = 0; i < RTE_DIM(mlx5_classes); i++) {
-		if (strcmp(class_name, mlx5_classes[i].name) == 0)
-			return mlx5_classes[i].driver_class;
-	}
-	return -EINVAL;
-}
-
-static struct mlx5_pci_driver *
-driver_get(uint32_t class)
-{
-	struct mlx5_pci_driver *driver;
-
-	TAILQ_FOREACH(driver, &drv_list, next) {
-		if (driver->driver_class == class)
-			return driver;
-	}
-	return NULL;
-}
-
-static int
-bus_cmdline_options_handler(__rte_unused const char *key,
-			    const char *class_names, void *opaque)
-{
-	int *ret = opaque;
-	char *nstr_org;
-	int class_val;
-	char *found;
-	char *nstr;
-	char *refstr = NULL;
-
-	*ret = 0;
-	nstr = strdup(class_names);
-	if (!nstr) {
-		*ret = -ENOMEM;
-		return *ret;
-	}
-	nstr_org = nstr;
-	found = strtok_r(nstr, ":", &refstr);
-	if (!found)
-		goto err;
-	do {
-		/* Extract each individual class name. Multiple
-		 * class key,value is supplied as class=net:vdpa:foo:bar.
-		 */
-		class_val = class_name_to_value(found);
-		/* Check if its a valid class. */
-		if (class_val < 0) {
-			*ret = -EINVAL;
-			goto err;
-		}
-		*ret |= class_val;
-		found = strtok_r(NULL, ":", &refstr);
-	} while (found);
-err:
-	free(nstr_org);
-	if (*ret < 0)
-		DRV_LOG(ERR, "Invalid mlx5 class options %s."
-			" Maybe typo in device class argument setting?",
-			class_names);
-	return *ret;
-}
-
-static int
-parse_class_options(const struct rte_devargs *devargs)
-{
-	const char *key = RTE_DEVARGS_KEY_CLASS;
-	struct rte_kvargs *kvlist;
-	int ret = 0;
-
-	if (devargs == NULL)
-		return 0;
-	kvlist = rte_kvargs_parse(devargs->args, NULL);
-	if (kvlist == NULL)
-		return 0;
-	if (rte_kvargs_count(kvlist, key))
-		rte_kvargs_process(kvlist, key, bus_cmdline_options_handler,
-				   &ret);
-	rte_kvargs_free(kvlist);
-	return ret;
-}
-
-static bool
-mlx5_bus_match(const struct mlx5_pci_driver *drv,
-	       const struct rte_pci_device *pci_dev)
-{
-	const struct rte_pci_id *id_table;
-
-	for (id_table = drv->pci_driver.id_table; id_table->vendor_id != 0;
-	     id_table++) {
-		/* Check if device's ids match the class driver's ids. */
-		if (id_table->vendor_id != pci_dev->id.vendor_id &&
-		    id_table->vendor_id != RTE_PCI_ANY_ID)
-			continue;
-		if (id_table->device_id != pci_dev->id.device_id &&
-		    id_table->device_id != RTE_PCI_ANY_ID)
-			continue;
-		if (id_table->subsystem_vendor_id !=
-		    pci_dev->id.subsystem_vendor_id &&
-		    id_table->subsystem_vendor_id != RTE_PCI_ANY_ID)
-			continue;
-		if (id_table->subsystem_device_id !=
-		    pci_dev->id.subsystem_device_id &&
-		    id_table->subsystem_device_id != RTE_PCI_ANY_ID)
-			continue;
-		if (id_table->class_id != pci_dev->id.class_id &&
-		    id_table->class_id != RTE_CLASS_ANY_ID)
-			continue;
-		return true;
-	}
-	return false;
-}
-
-static int
-is_valid_class_combination(uint32_t user_classes)
-{
-	unsigned int i;
-
-	/* Verify if user specified valid supported combination. */
-	for (i = 0; i < RTE_DIM(mlx5_class_combinations); i++) {
-		if (mlx5_class_combinations[i] == user_classes)
-			return 0;
-	}
-	/* Not found any valid class combination. */
-	return -EINVAL;
-}
-
-static struct mlx5_pci_device *
-pci_to_mlx5_device(const struct rte_pci_device *pci_dev)
-{
-	struct mlx5_pci_device *dev;
-
-	TAILQ_FOREACH(dev, &devices_list, next) {
-		if (dev->pci_dev == pci_dev)
-			return dev;
-	}
-	return NULL;
-}
-
-static bool
-device_class_enabled(const struct mlx5_pci_device *device, uint32_t class)
-{
-	return (device->classes_loaded & class) ? true : false;
-}
-
-static void
-dev_release(struct mlx5_pci_device *dev)
-{
-	TAILQ_REMOVE(&devices_list, dev, next);
-	rte_free(dev);
-}
-
-static int
-drivers_remove(struct mlx5_pci_device *dev, uint32_t enabled_classes)
-{
-	struct mlx5_pci_driver *driver;
-	int local_ret = -ENODEV;
-	unsigned int i = 0;
-	int ret = 0;
-
-	enabled_classes &= dev->classes_loaded;
-	while (enabled_classes) {
-		driver = driver_get(RTE_BIT64(i));
-		if (driver) {
-			local_ret = driver->pci_driver.remove(dev->pci_dev);
-			if (!local_ret)
-				dev->classes_loaded &= ~RTE_BIT64(i);
-			else if (ret == 0)
-				ret = local_ret;
-		}
-		enabled_classes &= ~RTE_BIT64(i);
-		i++;
-	}
-	if (local_ret)
-		ret = local_ret;
-	return ret;
-}
-
-static int
-drivers_probe(struct mlx5_pci_device *dev, struct rte_pci_driver *pci_drv,
-	      struct rte_pci_device *pci_dev, uint32_t user_classes)
-{
-	struct mlx5_pci_driver *driver;
-	uint32_t enabled_classes = 0;
-	bool already_loaded;
-	int ret;
-
-	TAILQ_FOREACH(driver, &drv_list, next) {
-		if ((driver->driver_class & user_classes) == 0)
-			continue;
-		if (!mlx5_bus_match(driver, pci_dev))
-			continue;
-		already_loaded = dev->classes_loaded & driver->driver_class;
-		if (already_loaded &&
-		    !(driver->pci_driver.drv_flags & RTE_PCI_DRV_PROBE_AGAIN)) {
-			DRV_LOG(ERR, "Device %s is already probed",
-				pci_dev->device.name);
-			ret = -EEXIST;
-			goto probe_err;
-		}
-		ret = driver->pci_driver.probe(pci_drv, pci_dev);
-		if (ret < 0) {
-			DRV_LOG(ERR, "Failed to load driver %s",
-				driver->pci_driver.driver.name);
-			goto probe_err;
-		}
-		enabled_classes |= driver->driver_class;
-	}
-	dev->classes_loaded |= enabled_classes;
-	return 0;
-probe_err:
-	/* Only unload drivers which are enabled which were enabled
-	 * in this probe instance.
-	 */
-	drivers_remove(dev, enabled_classes);
-	return ret;
-}
-
-/**
- * DPDK callback to register to probe multiple drivers for a PCI device.
- *
- * @param[in] pci_drv
- *   PCI driver structure.
- * @param[in] dev
- *   PCI device information.
- *
- * @return
- *   0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
-	       struct rte_pci_device *pci_dev)
-{
-	struct mlx5_pci_device *dev;
-	uint32_t user_classes = 0;
-	bool new_device = false;
-	int ret;
-
-	ret = parse_class_options(pci_dev->device.devargs);
-	if (ret < 0)
-		return ret;
-	user_classes = ret;
-	if (user_classes) {
-		/* Validate combination here. */
-		ret = is_valid_class_combination(user_classes);
-		if (ret) {
-			DRV_LOG(ERR, "Unsupported mlx5 classes supplied.");
-			return ret;
-		}
-	} else {
-		/* Default to net class. */
-		user_classes = MLX5_CLASS_ETH;
-	}
-	dev = pci_to_mlx5_device(pci_dev);
-	if (!dev) {
-		dev = rte_zmalloc("mlx5_pci_device", sizeof(*dev), 0);
-		if (!dev)
-			return -ENOMEM;
-		dev->pci_dev = pci_dev;
-		TAILQ_INSERT_HEAD(&devices_list, dev, next);
-		new_device = true;
-	}
-	ret = drivers_probe(dev, pci_drv, pci_dev, user_classes);
-	if (ret)
-		goto class_err;
-	return 0;
-class_err:
-	if (new_device)
-		dev_release(dev);
-	return ret;
-}
-
-/**
- * DPDK callback to remove one or more drivers for a PCI device.
- *
- * This function removes all drivers probed for a given PCI device.
- *
- * @param[in] pci_dev
- *   Pointer to the PCI device.
- *
- * @return
- *   0 on success, the function cannot fail.
- */
-static int
-mlx5_pci_remove(struct rte_pci_device *pci_dev)
-{
-	struct mlx5_pci_device *dev;
-	int ret;
-
-	dev = pci_to_mlx5_device(pci_dev);
-	if (!dev)
-		return -ENODEV;
-	/* Matching device found, cleanup and unload drivers. */
-	ret = drivers_remove(dev, dev->classes_loaded);
-	if (!ret)
-		dev_release(dev);
-	return ret;
-}
-
-static int
-mlx5_pci_dma_map(struct rte_pci_device *pci_dev, void *addr,
-		 uint64_t iova, size_t len)
-{
-	struct mlx5_pci_driver *driver = NULL;
-	struct mlx5_pci_driver *temp;
-	struct mlx5_pci_device *dev;
-	int ret = -EINVAL;
-
-	dev = pci_to_mlx5_device(pci_dev);
-	if (!dev)
-		return -ENODEV;
-	TAILQ_FOREACH(driver, &drv_list, next) {
-		if (device_class_enabled(dev, driver->driver_class) &&
-		    driver->pci_driver.dma_map) {
-			ret = driver->pci_driver.dma_map(pci_dev, addr,
-							 iova, len);
-			if (ret)
-				goto map_err;
-		}
-	}
-	return ret;
-map_err:
-	TAILQ_FOREACH(temp, &drv_list, next) {
-		if (temp == driver)
-			break;
-		if (device_class_enabled(dev, temp->driver_class) &&
-		    temp->pci_driver.dma_map && temp->pci_driver.dma_unmap)
-			temp->pci_driver.dma_unmap(pci_dev, addr, iova, len);
-	}
-	return ret;
-}
-
-static int
-mlx5_pci_dma_unmap(struct rte_pci_device *pci_dev, void *addr,
-		   uint64_t iova, size_t len)
-{
-	struct mlx5_pci_driver *driver;
-	struct mlx5_pci_device *dev;
-	int local_ret = -EINVAL;
-	int ret;
-
-	dev = pci_to_mlx5_device(pci_dev);
-	if (!dev)
-		return -ENODEV;
-	ret = 0;
-	/* There is no unmap error recovery in current implementation. */
-	TAILQ_FOREACH_REVERSE(driver, &drv_list, mlx5_pci_bus_drv_head, next) {
-		if (device_class_enabled(dev, driver->driver_class) &&
-		    driver->pci_driver.dma_unmap) {
-			local_ret = driver->pci_driver.dma_unmap(pci_dev, addr,
-								 iova, len);
-			if (local_ret && (ret == 0))
-				ret = local_ret;
-		}
-	}
-	if (local_ret)
-		ret = local_ret;
-	return ret;
-}
-
 /* PCI ID table is build dynamically based on registered mlx5 drivers. */
 static struct rte_pci_id *mlx5_pci_id_table;
 
-static struct rte_pci_driver mlx5_pci_driver = {
-	.driver = {
-		.name = MLX5_PCI_DRIVER_NAME,
-	},
-	.probe = mlx5_pci_probe,
-	.remove = mlx5_pci_remove,
-	.dma_map = mlx5_pci_dma_map,
-	.dma_unmap = mlx5_pci_dma_unmap,
-};
-
 static int
 pci_id_table_size_get(const struct rte_pci_id *id_table)
 {
@@ -511,7 +95,6 @@ pci_ids_table_update(const struct rte_pci_id *driver_id_table)
 	}
 	/* Terminate table with empty entry. */
 	updated_table[i].vendor_id = 0;
-	mlx5_pci_driver.id_table = updated_table;
 	mlx5_common_pci_driver.id_table = updated_table;
 	mlx5_pci_id_table = updated_table;
 	if (old_table)
@@ -519,20 +102,6 @@ pci_ids_table_update(const struct rte_pci_id *driver_id_table)
 	return 0;
 }
 
-void
-mlx5_pci_driver_register(struct mlx5_pci_driver *driver)
-{
-	int ret;
-
-	ret = pci_ids_table_update(driver->pci_driver.id_table);
-	if (ret)
-		return;
-	mlx5_pci_driver.drv_flags |= driver->pci_driver.drv_flags;
-	TAILQ_INSERT_TAIL(&drv_list, driver, next);
-}
-
-/********** New common PCI bus driver ********/
-
 bool
 mlx5_dev_is_pci(const struct rte_device *dev)
 {
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
deleted file mode 100644
index de89bb98bc..0000000000
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2020 Mellanox Technologies, Ltd
- */
-
-#ifndef _MLX5_COMMON_PCI_H_
-#define _MLX5_COMMON_PCI_H_
-
-/**
- * @file
- *
- * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
- *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
- *        \              \_--------------_/              /
- *         \_______________|   mlx5     |_______________/
- *                         | pci common |
- *                         --------------
- *                               |
- *                           -----------
- *                           |   mlx5  |
- *                           | pci dev |
- *                           -----------
- *
- * - mlx5 pci driver binds to mlx5 PCI devices defined by PCI
- *   ID table of all related mlx5 PCI devices.
- * - mlx5 class driver such as net, vdpa, regex PMD defines its
- *   specific PCI ID table and mlx5 bus driver probes matching
- *   class drivers.
- * - mlx5 pci bus driver is cental place that validates supported
- *   class combinations.
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-
-#include <rte_pci.h>
-#include <rte_bus_pci.h>
-
-#include <mlx5_common.h>
-
-void mlx5_common_pci_init(void);
-
-/**
- * A structure describing a mlx5 pci driver.
- */
-struct mlx5_pci_driver {
-	struct rte_pci_driver pci_driver;	/**< Inherit core pci driver. */
-	uint32_t driver_class;	/**< Class of this driver, enum mlx5_class */
-	TAILQ_ENTRY(mlx5_pci_driver) next;
-};
-
-/**
- * Register a mlx5_pci device driver.
- *
- * @param driver
- *   A pointer to a mlx5_pci_driver structure describing the driver
- *   to be registered.
- */
-__rte_internal
-void
-mlx5_pci_driver_register(struct mlx5_pci_driver *driver);
-
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-
-#endif /* _MLX5_COMMON_PCI_H_ */
diff --git a/drivers/common/mlx5/mlx5_common_private.h b/drivers/common/mlx5/mlx5_common_private.h
index 1096fa85e7..c929840408 100644
--- a/drivers/common/mlx5/mlx5_common_private.h
+++ b/drivers/common/mlx5/mlx5_common_private.h
@@ -31,6 +31,7 @@ int mlx5_common_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
 
 /* Common PCI bus driver: */
 
+void mlx5_common_pci_init(void);
 void mlx5_common_driver_on_register_pci(struct mlx5_class_driver *driver);
 bool mlx5_dev_pci_match(const struct mlx5_class_driver *drv,
 			const struct rte_device *dev);
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index a47c86e354..da91d167b5 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -138,13 +138,10 @@ INTERNAL {
 	mlx5_nl_vlan_vmwa_create; # WINDOWS_NO_EXPORT
 	mlx5_nl_vlan_vmwa_delete; # WINDOWS_NO_EXPORT
 
-	mlx5_pci_driver_register;
-
 	mlx5_os_alloc_pd;
 	mlx5_os_dealloc_pd;
 	mlx5_os_dereg_mr;
 	mlx5_os_get_ibv_dev; # WINDOWS_NO_EXPORT
-	mlx5_os_get_ibv_device; # WINDOWS_NO_EXPORT
 	mlx5_os_reg_mr;
 	mlx5_os_umem_dereg;
 	mlx5_os_umem_reg;
-- 
2.25.1