[dpdk-dev] [RFC 04/14] net/mlx5: migrate to bus-agnostic common driver

[Xueming Li <xuemingl@nvidia.com>]

To support SubFunction based on auxiliary bus, common driver supports
new bus-agnostic driver.

This patch migrates net driver to new common driver.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 46 +++++++++++++++++++++---------
 drivers/net/mlx5/linux/mlx5_os.h   |  3 --
 drivers/net/mlx5/mlx5.c            | 42 ++++++++++++---------------
 drivers/net/mlx5/mlx5.h            |  3 +-
 drivers/net/mlx5/mlx5_mr.c         | 36 +++++++++++------------
 drivers/net/mlx5/mlx5_rxtx.h       |  9 +++---
 drivers/net/mlx5/windows/mlx5_os.c | 14 ++++-----
 7 files changed, 80 insertions(+), 73 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index e8a97d4337..e8e6b0d5c9 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1977,14 +1977,6 @@ mlx5_os_pci_probe_pf(struct rte_pci_device *pci_dev,
 	struct mlx5_bond_info bond_info;
 	int ret = -1;
 
-	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
-		mlx5_pmd_socket_init();
-	ret = mlx5_init_once();
-	if (ret) {
-		DRV_LOG(ERR, "unable to init PMD global data: %s",
-			strerror(rte_errno));
-		return -rte_errno;
-	}
 	errno = 0;
 	ibv_list = mlx5_glue->get_device_list(&ret);
 	if (!ibv_list) {
@@ -2417,21 +2409,18 @@ mlx5_os_pci_probe_pf(struct rte_pci_device *pci_dev,
 }
 
 /**
- * DPDK callback to register a PCI device.
+ * Callback to register a PCI device.
  *
  * This function spawns Ethernet devices out of a given PCI device.
  *
- * @param[in] pci_drv
- *   PCI driver structure (mlx5_driver).
  * @param[in] pci_dev
  *   PCI device information.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-int
-mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
-		  struct rte_pci_device *pci_dev)
+static int
+mlx5_os_pci_probe(struct rte_pci_device *pci_dev)
 {
 	struct rte_eth_devargs eth_da = { .type = RTE_ETH_REPRESENTOR_NONE };
 	int ret = 0;
@@ -2470,6 +2459,35 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	return ret;
 }
 
+/**
+ * Common bus driver callback to probe a device.
+ *
+ * This function probe PCI bus device(s).
+ *
+ * @param[in] dev
+ *   Pointer to the generic device.
+ *
+ * @return
+ *   0 on success, the function cannot fail.
+ */
+int
+mlx5_os_net_probe(struct rte_device *dev)
+{
+	int ret;
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+		mlx5_pmd_socket_init();
+	ret = mlx5_init_once();
+	if (ret) {
+		DRV_LOG(ERR, "unable to init PMD global data: %s",
+			strerror(rte_errno));
+		return -rte_errno;
+	}
+	if (mlx5_dev_is_pci(dev))
+		return mlx5_os_pci_probe(RTE_DEV_TO_PCI(dev));
+	return 0;
+}
+
 static int
 mlx5_config_doorbell_mapping_env(const struct mlx5_dev_config *config)
 {
diff --git a/drivers/net/mlx5/linux/mlx5_os.h b/drivers/net/mlx5/linux/mlx5_os.h
index 4ae7d0ef47..af7cbeb418 100644
--- a/drivers/net/mlx5/linux/mlx5_os.h
+++ b/drivers/net/mlx5/linux/mlx5_os.h
@@ -19,7 +19,4 @@ enum {
 
 #define MLX5_NAMESIZE IF_NAMESIZE
 
-#define PCI_DRV_FLAGS  (RTE_PCI_DRV_INTR_LSC | \
-			RTE_PCI_DRV_INTR_RMV | \
-			RTE_PCI_DRV_PROBE_AGAIN)
 #endif /* RTE_PMD_MLX5_OS_H_ */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 95ac43268b..3defdb2db3 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -12,9 +12,7 @@
 
 #include <rte_malloc.h>
 #include <ethdev_driver.h>
-#include <ethdev_pci.h>
 #include <rte_pci.h>
-#include <rte_bus_pci.h>
 #include <rte_common.h>
 #include <rte_kvargs.h>
 #include <rte_rwlock.h>
@@ -22,13 +20,13 @@
 #include <rte_string_fns.h>
 #include <rte_alarm.h>
 #include <rte_cycles.h>
+#include <rte_bus_pci.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_devx_cmds.h>
 #include <mlx5_common.h>
 #include <mlx5_common_os.h>
 #include <mlx5_common_mp.h>
-#include <mlx5_common_pci.h>
 #include <mlx5_malloc.h>
 
 #include "mlx5_defs.h"
@@ -2335,23 +2333,23 @@ mlx5_eth_find_next(uint16_t port_id, struct rte_eth_dev *odev)
 }
 
 /**
- * DPDK callback to remove a PCI device.
+ * Callback to remove a device.
  *
- * This function removes all Ethernet devices belong to a given PCI device.
+ * This function removes all Ethernet devices belong to a given device.
  *
- * @param[in] pci_dev
- *   Pointer to the PCI device.
+ * @param[in] dev
+ *   Pointer to the generic device.
  *
  * @return
  *   0 on success, the function cannot fail.
  */
 static int
-mlx5_pci_remove(struct rte_pci_device *pci_dev)
+mlx5_net_remove(struct rte_device *dev)
 {
 	uint16_t port_id;
 	int ret = 0;
 
-	RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) {
+	RTE_ETH_FOREACH_DEV_OF(port_id, dev) {
 		/*
 		 * mlx5_dev_close() is not registered to secondary process,
 		 * call the close function explicitly for secondary process.
@@ -2442,19 +2440,17 @@ static const struct rte_pci_id mlx5_pci_id_map[] = {
 	}
 };
 
-static struct mlx5_pci_driver mlx5_driver = {
-	.driver_class = MLX5_CLASS_NET,
-	.pci_driver = {
-		.driver = {
-			.name = MLX5_PCI_DRIVER_NAME,
-		},
-		.id_table = mlx5_pci_id_map,
-		.probe = mlx5_os_pci_probe,
-		.remove = mlx5_pci_remove,
-		.dma_map = mlx5_dma_map,
-		.dma_unmap = mlx5_dma_unmap,
-		.drv_flags = PCI_DRV_FLAGS,
-	},
+static struct mlx5_class_driver mlx5_net_driver = {
+	.drv_class = MLX5_CLASS_NET,
+	.name = "mlx5_eth",
+	.id_table = mlx5_pci_id_map,
+	.probe = mlx5_os_net_probe,
+	.remove = mlx5_net_remove,
+	.dma_map = mlx5_net_dma_map,
+	.dma_unmap = mlx5_net_dma_unmap,
+	.probe_again = 1,
+	.intr_lsc = 1,
+	.intr_rmv = 1,
 };
 
 /* Initialize driver log type. */
@@ -2472,7 +2468,7 @@ RTE_INIT(rte_mlx5_pmd_init)
 	mlx5_set_cksum_table();
 	mlx5_set_swp_types_table();
 	if (mlx5_glue)
-		mlx5_pci_driver_register(&mlx5_driver);
+		mlx5_class_driver_register(&mlx5_net_driver);
 }
 
 RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 29a9b18887..27bb34e827 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1680,8 +1680,7 @@ int mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
 			 const struct mlx5_dev_config *config,
 			 struct mlx5_dev_ctx_shared *sh);
 int mlx5_os_get_pdn(void *pd, uint32_t *pdn);
-int mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
-		       struct rte_pci_device *pci_dev);
+int mlx5_os_net_probe(struct rte_device *dev);
 void mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh);
 void mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh);
 void mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index fcb475582d..2bce302eb5 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -7,7 +7,6 @@
 #include <rte_mempool.h>
 #include <rte_malloc.h>
 #include <rte_rwlock.h>
-#include <rte_bus_pci.h>
 
 #include <mlx5_common_mp.h>
 #include <mlx5_common_mr.h>
@@ -313,10 +312,10 @@ dev_to_eth_dev(struct rte_device *dev)
 }
 
 /**
- * DPDK callback to DMA map external memory to a PCI device.
+ * Callback to DMA map external memory to a device.
  *
- * @param pdev
- *   Pointer to the PCI device.
+ * @param rte_dev
+ *   Pointer to the generic device.
  * @param addr
  *   Starting virtual address of memory to be mapped.
  * @param iova
@@ -328,18 +327,18 @@ dev_to_eth_dev(struct rte_device *dev)
  *   0 on success, negative value on error.
  */
 int
-mlx5_dma_map(struct rte_pci_device *pdev, void *addr,
-	     uint64_t iova __rte_unused, size_t len)
+mlx5_net_dma_map(struct rte_device *rte_dev, void *addr,
+		 uint64_t iova __rte_unused, size_t len)
 {
 	struct rte_eth_dev *dev;
 	struct mlx5_mr *mr;
 	struct mlx5_priv *priv;
 	struct mlx5_dev_ctx_shared *sh;
 
-	dev = dev_to_eth_dev(&pdev->device);
+	dev = dev_to_eth_dev(rte_dev);
 	if (!dev) {
 		DRV_LOG(WARNING, "unable to find matching ethdev "
-				 "to PCI device %p", (void *)pdev);
+				 "to device %s", rte_dev->name);
 		rte_errno = ENODEV;
 		return -1;
 	}
@@ -362,10 +361,10 @@ mlx5_dma_map(struct rte_pci_device *pdev, void *addr,
 }
 
 /**
- * DPDK callback to DMA unmap external memory to a PCI device.
+ * Callback to DMA unmap external memory to a device.
  *
- * @param pdev
- *   Pointer to the PCI device.
+ * @param rte_dev
+ *   Pointer to the generic device.
  * @param addr
  *   Starting virtual address of memory to be unmapped.
  * @param iova
@@ -377,8 +376,8 @@ mlx5_dma_map(struct rte_pci_device *pdev, void *addr,
  *   0 on success, negative value on error.
  */
 int
-mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,
-	       uint64_t iova __rte_unused, size_t len __rte_unused)
+mlx5_net_dma_unmap(struct rte_device *rte_dev, void *addr,
+		   uint64_t iova __rte_unused, size_t len __rte_unused)
 {
 	struct rte_eth_dev *dev;
 	struct mlx5_priv *priv;
@@ -386,10 +385,10 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,
 	struct mlx5_mr *mr;
 	struct mr_cache_entry entry;
 
-	dev = dev_to_eth_dev(&pdev->device);
+	dev = dev_to_eth_dev(rte_dev);
 	if (!dev) {
-		DRV_LOG(WARNING, "unable to find matching ethdev "
-				 "to PCI device %p", (void *)pdev);
+		DRV_LOG(WARNING, "unable to find matching ethdev to device %s",
+			rte_dev->name);
 		rte_errno = ENODEV;
 		return -1;
 	}
@@ -399,9 +398,8 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,
 	mr = mlx5_mr_lookup_list(&sh->share_cache, &entry, (uintptr_t)addr);
 	if (!mr) {
 		rte_rwlock_read_unlock(&sh->share_cache.rwlock);
-		DRV_LOG(WARNING, "address 0x%" PRIxPTR " wasn't registered "
-				 "to PCI device %p", (uintptr_t)addr,
-				 (void *)pdev);
+		DRV_LOG(WARNING, "address 0x%" PRIxPTR " wasn't registered to device %s",
+			(uintptr_t)addr, rte_dev->name);
 		rte_errno = EINVAL;
 		return -1;
 	}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index e168dd46f9..ad1144e218 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -16,7 +16,6 @@
 #include <rte_hexdump.h>
 #include <rte_spinlock.h>
 #include <rte_io.h>
-#include <rte_bus_pci.h>
 #include <rte_cycles.h>
 
 #include <mlx5_common.h>
@@ -48,10 +47,10 @@ int mlx5_queue_state_modify(struct rte_eth_dev *dev,
 /* mlx5_mr.c */
 
 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
-int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova,
-		 size_t len);
-int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova,
-		   size_t len);
+int mlx5_net_dma_map(struct rte_device *rte_dev, void *addr, uint64_t iova,
+		     size_t len);
+int mlx5_net_dma_unmap(struct rte_device *rte_dev, void *addr, uint64_t iova,
+		       size_t len);
 
 /**
  * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 1e3260c6b5..2dfc957412 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -917,22 +917,22 @@ mlx5_match_devx_devices_to_addr(struct devx_device_bdf *devx_bdf,
 }
 
 /**
- * DPDK callback to register a PCI device.
+ * Driver callback to register a device.
  *
  * This function spawns Ethernet devices out of a given PCI device.
  *
- * @param[in] pci_drv
- *   PCI driver structure (mlx5_driver).
- * @param[in] pci_dev
- *   PCI device information.
+ * @param[in] rte_dev
+ *   Pointer to generic device.
+ * @param[in] ib_dev
+ *   Pointer to Verbs device.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
-		  struct rte_pci_device *pci_dev)
+mlx5_os_net_probe(struct rte_device *rte_dev)
 {
+	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(rte_dev);
 	struct devx_device_bdf *devx_bdf_devs, *orig_devx_bdf_devs;
 	/*
 	 * Number of found IB Devices matching with requested PCI BDF.
-- 
2.25.1