[dpdk-dev] [RFC 03/14] net/mlx5: remove PCI dependency

[Xueming Li <xuemingl@nvidia.com>]

To support more bus types, remove PCI dependency where possible.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_pci.c   |  6 +--
 drivers/net/mlx5/linux/mlx5_ethdev_os.c |  2 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  4 +-
 drivers/net/mlx5/mlx5.c                 | 53 ++++++++++++++++---------
 drivers/net/mlx5/mlx5.h                 |  8 ++--
 drivers/net/mlx5/mlx5_ethdev.c          |  2 +-
 drivers/net/mlx5/mlx5_mr.c              | 14 +++----
 drivers/net/mlx5/mlx5_trigger.c         | 12 +++---
 drivers/net/mlx5/mlx5_txq.c             |  2 +-
 drivers/net/mlx5/windows/mlx5_os.c      |  6 +--
 10 files changed, 58 insertions(+), 51 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 53090173a2..5a824dd50f 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -534,11 +534,7 @@ mlx5_pci_driver_register(struct mlx5_pci_driver *driver)
 bool
 mlx5_dev_is_pci(const struct rte_device *dev)
 {
-	struct rte_devargs *da = dev->devargs;
-
-	if (da == NULL || da->bus == NULL)
-		return false;
-	return strcmp(da->bus->name, "pci") == 0;
+	return strcmp(dev->bus->name, "pci") == 0;
 }
 
 struct ibv_device *
diff --git a/drivers/net/mlx5/linux/mlx5_ethdev_os.c b/drivers/net/mlx5/linux/mlx5_ethdev_os.c
index ddc1371aa9..6fdb310129 100644
--- a/drivers/net/mlx5/linux/mlx5_ethdev_os.c
+++ b/drivers/net/mlx5/linux/mlx5_ethdev_os.c
@@ -346,7 +346,7 @@ mlx5_find_master_dev(struct rte_eth_dev *dev)
 	priv = dev->data->dev_private;
 	domain_id = priv->domain_id;
 	MLX5_ASSERT(priv->representor);
-	MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+	MLX5_ETH_FOREACH_DEV(port_id, dev) {
 		struct mlx5_priv *opriv =
 			rte_eth_devices[port_id].data->dev_private;
 		if (opriv &&
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 534a56a555..e8a97d4337 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -925,6 +925,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	sh = mlx5_alloc_shared_dev_ctx(spawn, config);
 	if (!sh)
 		return NULL;
+	sh->numa_node = dpdk_dev->numa_node;
 	config->devx = sh->devx;
 #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
 	config->dest_tir = 1;
@@ -1133,7 +1134,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	 * Look for sibling devices in order to reuse their switch domain
 	 * if any, otherwise allocate one.
 	 */
-	MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
 		const struct mlx5_priv *opriv =
 			rte_eth_devices[port_id].data->dev_private;
 
@@ -2556,7 +2557,6 @@ mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
 	int dbmap_env;
 	int err = 0;
 
-	sh->numa_node = spawn->pci_dev->device.numa_node;
 	pthread_mutex_init(&sh->txpp.mutex, NULL);
 	/*
 	 * Configure environment variable "MLX5_BF_SHUT_UP"
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index d0faa45944..95ac43268b 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1185,7 +1185,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	 */
 	err = mlx5_mr_btree_init(&sh->share_cache.cache,
 				 MLX5_MR_BTREE_CACHE_N * 2,
-				 spawn->pci_dev->device.numa_node);
+				 sh->numa_node);
 	if (err) {
 		err = rte_errno;
 		goto error;
@@ -1620,7 +1620,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		unsigned int c = 0;
 		uint16_t port_id;
 
-		MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+		MLX5_ETH_FOREACH_DEV(port_id, dev) {
 			struct mlx5_priv *opriv =
 				rte_eth_devices[port_id].data->dev_private;
 
@@ -2057,7 +2057,8 @@ void
 mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
 		    struct mlx5_dev_config *config)
 {
-	if (config->txq_inline_min != MLX5_ARG_UNSET) {
+	if (config->txq_inline_min != MLX5_ARG_UNSET &&
+	    spawn->pci_dev != NULL) {
 		/* Application defines size of inlined data explicitly. */
 		switch (spawn->pci_dev->id.device_id) {
 		case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
@@ -2124,6 +2125,11 @@ mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
 			}
 		}
 	}
+	if (spawn->pci_dev == NULL) {
+		if (config->txq_inline_min == MLX5_ARG_UNSET)
+			config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
+		goto exit;
+	}
 	/*
 	 * We get here if we are unable to deduce
 	 * inline data size with DevX. Try PCI ID
@@ -2258,7 +2264,7 @@ mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
 	if (sh->refcnt == 1)
 		return 0;
 	/* Find the device with shared context. */
-	MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
 		struct mlx5_priv *opriv =
 			rte_eth_devices[port_id].data->dev_private;
 
@@ -2286,33 +2292,42 @@ mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
 
 /**
  * Look for the ethernet device belonging to mlx5 driver.
+ * If device specified, look for ports belong to same PCI/bonding.
  *
  * @param[in] port_id
  *   port_id to start looking for device.
- * @param[in] pci_dev
- *   Pointer to the hint PCI device. When device is being probed
- *   the its siblings (master and preceding representors might
- *   not have assigned driver yet (because the mlx5_os_pci_probe()
- *   is not completed yet, for this case match on hint PCI
- *   device may be used to detect sibling device.
+ * @param[in] odev
+ *   Device to detect sibling.
  *
  * @return
  *   port_id of found device, RTE_MAX_ETHPORT if not found.
  */
 uint16_t
-mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev)
+mlx5_eth_find_next(uint16_t port_id, struct rte_eth_dev *odev)
 {
-	while (port_id < RTE_MAX_ETHPORTS) {
+	const struct mlx5_priv *opriv = NULL;
+
+	if (odev)
+		opriv = odev->data->dev_private;
+	for ( ; port_id < RTE_MAX_ETHPORTS; port_id++) {
 		struct rte_eth_dev *dev = &rte_eth_devices[port_id];
+		const struct mlx5_priv *priv;
 
-		if (dev->state != RTE_ETH_DEV_UNUSED &&
-		    dev->device &&
-		    (dev->device == &pci_dev->device ||
-		     (dev->device->driver &&
-		     dev->device->driver->name &&
-		     !strcmp(dev->device->driver->name, MLX5_PCI_DRIVER_NAME))))
+		if (dev->state == RTE_ETH_DEV_UNUSED)
+			continue;
+		priv = dev->data->dev_private;
+		if (odev != NULL) {
+			/* odev specified, find devices on same PCI/bonding. */
+			if (opriv->sh == priv->sh ||
+			    odev->device == dev->device)
+				break;
+		} else if (dev->device != NULL && dev->device->driver &&
+			dev->device->driver->name &&
+			!strcmp(dev->device->driver->name,
+				MLX5_PCI_DRIVER_NAME)) {
+			/* odev not specified, found all mlx5 devices. */
 			break;
-		port_id++;
+		}
 	}
 	if (port_id >= RTE_MAX_ETHPORTS)
 		return RTE_MAX_ETHPORTS;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 32b2817bf2..29a9b18887 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1404,16 +1404,16 @@ int mlx5_proc_priv_init(struct rte_eth_dev *dev);
 void mlx5_proc_priv_uninit(struct rte_eth_dev *dev);
 int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev,
 			      struct rte_eth_udp_tunnel *udp_tunnel);
-uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev);
+uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_eth_dev *odev);
 int mlx5_dev_close(struct rte_eth_dev *dev);
 bool mlx5_is_hpf(struct rte_eth_dev *dev);
 void mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh);
 
 /* Macro to iterate over all valid ports for mlx5 driver. */
-#define MLX5_ETH_FOREACH_DEV(port_id, pci_dev) \
-	for (port_id = mlx5_eth_find_next(0, pci_dev); \
+#define MLX5_ETH_FOREACH_DEV(port_id, dev) \
+	for (port_id = mlx5_eth_find_next(0, dev); \
 	     port_id < RTE_MAX_ETHPORTS; \
-	     port_id = mlx5_eth_find_next(port_id + 1, pci_dev))
+	     port_id = mlx5_eth_find_next(port_id + 1, dev))
 int mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs);
 struct mlx5_dev_ctx_shared *
 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 90baee5aa4..4654b85844 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -335,7 +335,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 	if (priv->representor) {
 		uint16_t port_id;
 
-		MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+		MLX5_ETH_FOREACH_DEV(port_id, dev) {
 			struct mlx5_priv *opriv =
 				rte_eth_devices[port_id].data->dev_private;
 
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index e791b6338d..fcb475582d 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -290,23 +290,23 @@ mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,
 }
 
 /**
- * Finds the first ethdev that match the pci device.
+ * Finds the first ethdev that match the device.
  * The existence of multiple ethdev per pci device is only with representors.
  * On such case, it is enough to get only one of the ports as they all share
  * the same ibv context.
  *
- * @param pdev
- *   Pointer to the PCI device.
+ * @param dev
+ *   Pointer to the device.
  *
  * @return
  *   Pointer to the ethdev if found, NULL otherwise.
  */
 static struct rte_eth_dev *
-pci_dev_to_eth_dev(struct rte_pci_device *pdev)
+dev_to_eth_dev(struct rte_device *dev)
 {
 	uint16_t port_id;
 
-	port_id = rte_eth_find_next_of(0, &pdev->device);
+	port_id = rte_eth_find_next_of(0, dev);
 	if (port_id == RTE_MAX_ETHPORTS)
 		return NULL;
 	return &rte_eth_devices[port_id];
@@ -336,7 +336,7 @@ mlx5_dma_map(struct rte_pci_device *pdev, void *addr,
 	struct mlx5_priv *priv;
 	struct mlx5_dev_ctx_shared *sh;
 
-	dev = pci_dev_to_eth_dev(pdev);
+	dev = dev_to_eth_dev(&pdev->device);
 	if (!dev) {
 		DRV_LOG(WARNING, "unable to find matching ethdev "
 				 "to PCI device %p", (void *)pdev);
@@ -386,7 +386,7 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,
 	struct mlx5_mr *mr;
 	struct mr_cache_entry entry;
 
-	dev = pci_dev_to_eth_dev(pdev);
+	dev = dev_to_eth_dev(&pdev->device);
 	if (!dev) {
 		DRV_LOG(WARNING, "unable to find matching ethdev "
 				 "to PCI device %p", (void *)pdev);
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ae7fcca229..6c8a64ce03 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -697,7 +697,7 @@ mlx5_hairpin_bind_single_port(struct rte_eth_dev *dev, uint16_t rx_port)
 	uint32_t explicit;
 	uint16_t rx_queue;
 
-	if (mlx5_eth_find_next(rx_port, priv->pci_dev) != rx_port) {
+	if (mlx5_eth_find_next(rx_port, dev) != rx_port) {
 		rte_errno = ENODEV;
 		DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port);
 		return -rte_errno;
@@ -835,7 +835,7 @@ mlx5_hairpin_unbind_single_port(struct rte_eth_dev *dev, uint16_t rx_port)
 	int ret;
 	uint16_t cur_port = priv->dev_data->port_id;
 
-	if (mlx5_eth_find_next(rx_port, priv->pci_dev) != rx_port) {
+	if (mlx5_eth_find_next(rx_port, dev) != rx_port) {
 		rte_errno = ENODEV;
 		DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port);
 		return -rte_errno;
@@ -893,7 +893,6 @@ mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port)
 {
 	int ret = 0;
 	uint16_t p, pp;
-	struct mlx5_priv *priv = dev->data->dev_private;
 
 	/*
 	 * If the Rx port has no hairpin configuration with the current port,
@@ -902,7 +901,7 @@ mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port)
 	 * information updating.
 	 */
 	if (rx_port == RTE_MAX_ETHPORTS) {
-		MLX5_ETH_FOREACH_DEV(p, priv->pci_dev) {
+		MLX5_ETH_FOREACH_DEV(p, dev) {
 			ret = mlx5_hairpin_bind_single_port(dev, p);
 			if (ret != 0)
 				goto unbind;
@@ -912,7 +911,7 @@ mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port)
 		return mlx5_hairpin_bind_single_port(dev, rx_port);
 	}
 unbind:
-	MLX5_ETH_FOREACH_DEV(pp, priv->pci_dev)
+	MLX5_ETH_FOREACH_DEV(pp, dev)
 		if (pp < p)
 			mlx5_hairpin_unbind_single_port(dev, pp);
 	return ret;
@@ -927,10 +926,9 @@ mlx5_hairpin_unbind(struct rte_eth_dev *dev, uint16_t rx_port)
 {
 	int ret = 0;
 	uint16_t p;
-	struct mlx5_priv *priv = dev->data->dev_private;
 
 	if (rx_port == RTE_MAX_ETHPORTS)
-		MLX5_ETH_FOREACH_DEV(p, priv->pci_dev) {
+		MLX5_ETH_FOREACH_DEV(p, dev) {
 			ret = mlx5_hairpin_unbind_single_port(dev, p);
 			if (ret != 0)
 				return ret;
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 3e5e94444b..f68c0c61a9 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -816,7 +816,7 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
 	if (config->txqs_inline == MLX5_ARG_UNSET)
 		txqs_inline =
 #if defined(RTE_ARCH_ARM64)
-		(priv->pci_dev->id.device_id ==
+		(priv->pci_dev && priv->pci_dev->id.device_id ==
 			PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) ?
 			MLX5_INLINE_MAX_TXQS_BLUEFIELD :
 #endif
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 3fe3f55f49..1e3260c6b5 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -229,9 +229,6 @@ mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
 	struct mlx5_context *mlx5_ctx;
 
 	pthread_mutex_init(&sh->txpp.mutex, NULL);
-	/* Set numa node from pci probe */
-	sh->numa_node = spawn->pci_dev->device.numa_node;
-
 	/* Try to open device with DevX */
 	rte_errno = 0;
 	sh->ctx = mlx5_glue->open_device(spawn->phys_dev);
@@ -344,6 +341,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	sh = mlx5_alloc_shared_dev_ctx(spawn, config);
 	if (!sh)
 		return NULL;
+	sh->numa_node = dpdk_dev->numa_node;
 	config->devx = sh->devx;
 	/* Initialize the shutdown event in mlx5_dev_spawn to
 	 * support mlx5_is_removed for Windows.
@@ -393,7 +391,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	 * Look for sibling devices in order to reuse their switch domain
 	 * if any, otherwise allocate one.
 	 */
-	MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
 		const struct mlx5_priv *opriv =
 			rte_eth_devices[port_id].data->dev_private;
 
-- 
2.25.1