[PATCH 1/2] common/mlx5: add device duplication function

[PATCH 1/2] common/mlx5: add device duplication function

[Gregory Etelson]

From: Michael Baum <michaelba@nvidia.com>

Add function for creating additional CTX for same device base on
existing CTX.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.c   | 52 ++++++++++++++++++++
 drivers/common/mlx5/mlx5_common.h            |  4 ++
 drivers/common/mlx5/version.map              |  1 +
 drivers/common/mlx5/windows/mlx5_common_os.c | 31 ++++++++++++
 4 files changed, 88 insertions(+)

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index 41345e1597..2f5032f0da 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -873,6 +873,58 @@ mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes)
 	return 0;
 }
 
+/**
+ * API function to obtain a new InfiniBand (IB) context for a given common device.
+ *
+ * This function provides a port-agnostic IB context for a physical device, enabling the
+ * device to create and manage resources that can be initialized when a port starts and
+ * released when another port stops.
+ *
+ * For Linux, it imports new context from the existing context.
+ *
+ * @param cdev
+ *   Pointer to the mlx5 device structure.
+ *
+ * @return
+ *   Pointer to an `ibv_context` on success, or NULL on failure, with `rte_errno` set.
+ */
+void *
+mlx5_os_get_physical_device_ctx(struct mlx5_common_device *cdev)
+{
+	struct ibv_context *ctx = NULL;
+	int cmd_fd = ((struct ibv_context *)cdev->ctx)->cmd_fd;
+	int new_cmd_fd;
+
+	/*
+	 * Duplicate the command FD to pass it as input to the import device function.
+	 * If the import function succeeds, the new device context takes ownership of
+	 * this FD, which will be freed when the new device is closed.
+	 * If the import function fails, we are responsible for closing this FD.
+	 */
+	new_cmd_fd = dup(cmd_fd);
+	if (new_cmd_fd < 0) {
+		DRV_LOG(ERR,
+			"Failed to duplicate FD %d for IB device \"%s\": %s",
+			cmd_fd, mlx5_os_get_ctx_device_name(cdev->ctx),
+			rte_strerror(errno));
+		rte_errno = errno;
+		return NULL;
+	}
+	/* Attempt to import the duplicated FD to create a new device context. */
+	ctx = mlx5_glue->import_device(new_cmd_fd);
+	if (!ctx) {
+		DRV_LOG(ERR, "Failed to import IB device \"%s\": %s",
+			mlx5_os_get_ctx_device_name(cdev->ctx),
+			rte_strerror(errno));
+		close(new_cmd_fd);
+		rte_errno = errno;
+		return NULL;
+	}
+	DRV_LOG(INFO, "IB device \"%s\" successfully imported, old_fd=%d, new_fd=%d",
+		mlx5_os_get_ctx_device_name(cdev->ctx), cmd_fd, new_cmd_fd);
+	return (void *)ctx;
+}
+
 int
 mlx5_get_device_guid(const struct rte_pci_addr *dev, uint8_t *guid, size_t len)
 {
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index e7bd4c6ec4..bea1382911 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -651,6 +651,10 @@ mlx5_devx_uar_release(struct mlx5_uar *uar);
 
 /* mlx5_common_os.c */
 
+__rte_internal
+void *
+mlx5_os_get_physical_device_ctx(struct mlx5_common_device *cdev);
+
 int mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes);
 int mlx5_os_pd_prepare(struct mlx5_common_device *cdev);
 int mlx5_os_pd_release(struct mlx5_common_device *cdev);
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 8301485839..819e6b96cb 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -150,6 +150,7 @@ INTERNAL {
 	mlx5_nl_rdma_monitor_info_get; # WINDOWS_NO_EXPORT
 	mlx5_nl_rdma_monitor_cap_get; # WINDOWS_NO_EXPORT
 
+	mlx5_os_get_physical_device_ctx;
 	mlx5_os_umem_dereg;
 	mlx5_os_umem_reg;
 
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index 2b6058eb3e..68f1981193 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -266,6 +266,37 @@ mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes)
 	return -rte_errno;
 }
 
+/**
+ * API function to obtain a new MLX5 context for a given common device.
+ *
+ * This function provides a port-agnostic context for a physical device, enabling the
+ * device to create and manage resources that can be initialized when a port starts and
+ * released when another port stops.
+ *
+ * For Windows, it creates a new context for the device regardless to existing context.
+ *
+ * @param cdev
+ *   Pointer to the mlx5 device structure.
+ *
+ * @return
+ *   Pointer to an `ibv_context` on success, or NULL on failure, with `rte_errno` set.
+ */
+void *
+mlx5_os_get_physical_device_ctx(struct mlx5_common_device *cdev)
+{
+	struct mlx5_common_device temp = {
+		.dev = cdev->dev,
+	};
+
+	if (mlx5_os_open_device(&temp, MLX5_CLASS_ETH) < 0) {
+		DRV_LOG(ERR, "Failed to duplicate DevX device \"%s\": %s",
+			mlx5_os_get_ctx_device_name(cdev->ctx),
+			rte_strerror(rte_errno));
+		return NULL;
+	}
+	return (void *)temp.ctx;
+}
+
 /**
  * Register umem.
  *
-- 
2.45.2

[PATCH 2/2] net/mlx5: fix GENEVE parser cleanup

[Gregory Etelson]

From: Michael Baum <michaelba@nvidia.com>

The GENEVE parser is shared across ports on the same physical device. It
is created once for the first port and increments a reference counter
for each additional port. The parser is created using the InfiniBand
(IBV) context (CTX) of the first port, and cleanup should use the same
context.

Previously, if the port owning the context closed while another port
still used the parser, the port closure would fail due to the shared
parser dependency.

This patch addresses the issue by changing the approach: the physical
device now creates its own distinct context by importing the context
from the first initialized port. This ensures that parser creation and
cleanup use a consistent context, independent of individual port
lifecycles.

Fixes: f5177bdc8b76 ("net/mlx5: add GENEVE TLV options parser API")
Cc: michaelba@nvidia.com

Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 drivers/net/mlx5/mlx5.c             | 21 ++++++------
 drivers/net/mlx5/mlx5.h             |  2 +-
 drivers/net/mlx5/mlx5_flow.h        |  2 --
 drivers/net/mlx5/mlx5_flow_geneve.c | 50 +++--------------------------
 4 files changed, 18 insertions(+), 57 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0f49cb5e5b..a16568ff22 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1724,6 +1724,16 @@ mlx5_get_physical_device(struct mlx5_common_device *cdev)
 		rte_errno = ENOMEM;
 		return NULL;
 	}
+	/*
+	 * The same CTX create the physical device objects should destroy them.
+	 * Since we can't be sure it will be done by same CTX, we prepare for
+	 * the physical device a special CTX used by objects creation.
+	 */
+	phdev->ctx = mlx5_os_get_physical_device_ctx(cdev);
+	if (!phdev->ctx) {
+		mlx5_free(phdev);
+		return NULL;
+	}
 	phdev->guid = attr->system_image_guid;
 	phdev->refcnt = 1;
 	LIST_INSERT_HEAD(&phdev_list, phdev, next);
@@ -1767,6 +1777,8 @@ mlx5_physical_device_destroy(struct mlx5_physical_device *phdev)
 		return;
 	/* Remove physical device from the global device list. */
 	LIST_REMOVE(phdev, next);
+	MLX5_ASSERT(phdev->ctx);
+	claim_zero(mlx5_glue->close_device(phdev->ctx));
 	mlx5_free(phdev);
 }
 
@@ -2323,15 +2335,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		rte_errno = EBUSY;
 		return -EBUSY;
 	}
-#ifdef HAVE_MLX5_HWS_SUPPORT
-	/* Check if shared GENEVE options created on context being closed. */
-	ret = mlx5_geneve_tlv_options_check_busy(priv);
-	if (ret) {
-		DRV_LOG(ERR, "port %u maintains shared GENEVE TLV options",
-			dev->data->port_id);
-		return ret;
-	}
-#endif
 	DRV_LOG(DEBUG, "port %u closing device \"%s\"",
 		dev->data->port_id, sh->ibdev_name);
 	/*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 6df99c25e2..0194887a8b 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1527,7 +1527,7 @@ struct mlx5_common_nic_config {
  */
 struct mlx5_physical_device {
 	LIST_ENTRY(mlx5_physical_device) next;
-	struct mlx5_dev_ctx_shared *sh; /* Created on sherd context. */
+	void *ctx; /* CTX for creation of options. */
 	uint64_t guid; /* System image guid, the uniq ID of physical device. */
 	struct mlx5_geneve_tlv_options *tlv_options;
 	struct mlx5_common_nic_config config;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index a5bde158ca..9816ed9238 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -3469,8 +3469,6 @@ flow_hw_resource_release(struct rte_eth_dev *dev);
 int
 mlx5_geneve_tlv_options_destroy(struct mlx5_geneve_tlv_options *options,
 				struct mlx5_physical_device *phdev);
-int
-mlx5_geneve_tlv_options_check_busy(struct mlx5_priv *priv);
 void
 flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable);
 int flow_dv_action_validate(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_geneve.c b/drivers/net/mlx5/mlx5_flow_geneve.c
index 6bf53e1270..4d57bb763f 100644
--- a/drivers/net/mlx5/mlx5_flow_geneve.c
+++ b/drivers/net/mlx5/mlx5_flow_geneve.c
@@ -590,8 +590,8 @@ mlx5_geneve_tlv_option_copy(struct rte_pmd_mlx5_geneve_tlv *dst,
 /**
  * Create list of GENEVE TLV options according to user configuration list.
  *
- * @param sh
- *   Shared context the options are being created on.
+ * @param ctx
+ *   Context returned from mlx5 open_device() glue function.
  * @param tlv_list
  *   A list of GENEVE TLV options to create parser for them.
  * @param nb_options
@@ -604,7 +604,7 @@ mlx5_geneve_tlv_option_copy(struct rte_pmd_mlx5_geneve_tlv *dst,
  *   NULL otherwise and rte_errno is set.
  */
 static struct mlx5_geneve_tlv_options *
-mlx5_geneve_tlv_options_create(struct mlx5_dev_ctx_shared *sh,
+mlx5_geneve_tlv_options_create(void *ctx,
 			       const struct rte_pmd_mlx5_geneve_tlv tlv_list[],
 			       uint8_t nb_options, uint8_t sample_id)
 {
@@ -625,7 +625,7 @@ mlx5_geneve_tlv_options_create(struct mlx5_dev_ctx_shared *sh,
 	}
 	for (i = 0; i < nb_options; ++i) {
 		spec = &tlv_list[i];
-		ret = mlx5_geneve_tlv_option_create(sh->cdev->ctx, spec,
+		ret = mlx5_geneve_tlv_option_create(ctx, spec,
 						    &options->options[i], sample_id);
 		if (ret < 0)
 			goto error;
@@ -633,8 +633,6 @@ mlx5_geneve_tlv_options_create(struct mlx5_dev_ctx_shared *sh,
 		data_mask = options->buffer + i * MAX_GENEVE_OPTION_DATA_SIZE;
 		mlx5_geneve_tlv_option_copy(&options->spec[i], spec, data_mask);
 	}
-	MLX5_ASSERT(sh->phdev->sh == NULL);
-	sh->phdev->sh = sh;
 	options->nb_options = nb_options;
 	options->refcnt = 1;
 	return options;
@@ -676,41 +674,9 @@ mlx5_geneve_tlv_options_destroy(struct mlx5_geneve_tlv_options *options,
 	}
 	mlx5_free(options);
 	phdev->tlv_options = NULL;
-	phdev->sh = NULL;
 	return 0;
 }
 
-/**
- * Check if GENEVE TLV options are hosted on the current port
- * and the port can be closed
- *
- * @param priv
- *   Device private data.
- *
- * @return
- *   0 on success, a negative EBUSY and rte_errno is set.
- */
-int
-mlx5_geneve_tlv_options_check_busy(struct mlx5_priv *priv)
-{
-	struct mlx5_physical_device *phdev = mlx5_get_locked_physical_device(priv);
-	struct mlx5_dev_ctx_shared *sh = priv->sh;
-
-	if (!phdev || phdev->sh != sh) {
-		mlx5_unlock_physical_device();
-		return 0;
-	}
-	if (!sh->phdev->tlv_options || sh->phdev->tlv_options->refcnt == 1) {
-		/* Mark port as being closed one */
-		sh->phdev->sh = NULL;
-		mlx5_unlock_physical_device();
-		return 0;
-	}
-	mlx5_unlock_physical_device();
-	rte_errno = EBUSY;
-	return -EBUSY;
-}
-
 /**
  * Validate GENEVE TLV option user request structure.
  *
@@ -955,18 +921,12 @@ mlx5_geneve_tlv_parser_create(uint16_t port_id,
 			rte_errno = EEXIST;
 			return NULL;
 		}
-		if (phdev->sh == NULL) {
-			mlx5_unlock_physical_device();
-			DRV_LOG(ERR, "GENEVE TLV options are hosted on port being closed.");
-			rte_errno = EBUSY;
-			return NULL;
-		}
 		/* Use existing options. */
 		options->refcnt++;
 		goto exit;
 	}
 	/* Create GENEVE TLV options for this physical device. */
-	options = mlx5_geneve_tlv_options_create(priv->sh, tlv_list, nb_options, sample_id);
+	options = mlx5_geneve_tlv_options_create(phdev->ctx, tlv_list, nb_options, sample_id);
 	if (!options) {
 		mlx5_unlock_physical_device();
 		return NULL;
-- 
2.45.2