[dpdk-dev] [PATCH] net/mlx5: optimize the device spawn time with representors

[dpdk-dev] [PATCH] net/mlx5: optimize the device spawn time with representors

[Jiawei Wang]

During the device spawn process, mlx5 PMD queried the available flow
priorities by calling mlx5_flow_discover_priorities, queried
if the DR drop action was supported on the root table by calling
the mlx5_flow_discover_dr_action_support routine, and queried the
availability of metadata register C by calling mlx5_flow_discover_mreg_c.

These functions created the test flows to get the supported fields, and
at the end destroyed the test flows. The test flows in the first two
functions was created on the root table.
If the device was spawned with multiple representors, these test flows
were created and destroyed on each representor as well. The above
operations took a significant amount of init time during the device spawn.

This patch optimizes the device discover functions, if there is
the device with multiple representors (VF/SF) being spawned,
the priority and drop action and metadata register support check can be
done only ones and check results can be shared for all representors.

Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 33 +++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.h            | 10 ++++++---
 drivers/net/mlx5/mlx5_flow.c       | 31 ++++++++++++++--------------
 drivers/net/mlx5/mlx5_flow_verbs.c |  4 ++--
 drivers/net/mlx5/windows/mlx5_os.c | 12 ++++++-----
 5 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 3746057673..1ee7f4110b 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -805,10 +805,15 @@ mlx5_flow_drop_action_config(struct rte_eth_dev *dev __rte_unused)
 	 * DR supports drop action placeholder when it is supported;
 	 * otherwise, use the queue drop action.
 	 */
-	if (mlx5_flow_discover_dr_action_support(dev))
-		priv->root_drop_action = priv->drop_queue.hrxq->action;
-	else
+	if (!priv->sh->drop_action_check_flag) {
+		if (!mlx5_flow_discover_dr_action_support(dev))
+			priv->sh->dr_drop_action_en = 1;
+		priv->sh->drop_action_check_flag = 1;
+	}
+	if (priv->sh->dr_drop_action_en)
 		priv->root_drop_action = priv->sh->dr_drop_action;
+	else
+		priv->root_drop_action = priv->drop_queue.hrxq->action;
 #endif
 }
 
@@ -1830,13 +1835,19 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	priv->drop_queue.hrxq = mlx5_drop_action_create(eth_dev);
 	if (!priv->drop_queue.hrxq)
 		goto error;
-	/* Supported Verbs flow priority number detection. */
-	err = mlx5_flow_discover_priorities(eth_dev);
+	/* Port representor shares the same max prioirity with master. */
+	if (!priv->sh->flow_priority_check_flag) {
+		/* Supported Verbs flow priority number detection. */
+		err = mlx5_flow_discover_priorities(eth_dev);
+		priv->sh->flow_max_priority = err;
+		priv->sh->flow_priority_check_flag = 1;
+	} else {
+		err = priv->sh->flow_max_priority;
+	}
 	if (err < 0) {
 		err = -err;
 		goto error;
 	}
-	priv->config.flow_prio = err;
 	if (!priv->config.dv_esw_en &&
 	    priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
 		DRV_LOG(WARNING, "metadata mode %u is not supported "
@@ -1862,10 +1873,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
-	err = mlx5_flow_discover_mreg_c(eth_dev);
-	if (err < 0) {
-		err = -err;
-		goto error;
+	if (!priv->sh->metadata_regc_check_flag) {
+		err = mlx5_flow_discover_mreg_c(eth_dev);
+		if (err < 0) {
+			err = -err;
+			goto error;
+		}
 	}
 	if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
 		DRV_LOG(DEBUG,
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3581414b78..661df5d377 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -283,9 +283,6 @@ struct mlx5_dev_config {
 	} mprq; /* Configurations for Multi-Packet RQ. */
 	int mps; /* Multi-packet send supported mode. */
 	int dbnc; /* Skip doorbell register write barrier. */
-	unsigned int flow_prio; /* Number of flow priorities. */
-	enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
-	/* Availibility of mreg_c's. */
 	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
 	unsigned int ind_table_max_size; /* Maximum indirection table size. */
 	unsigned int max_dump_files_num; /* Maximum dump files per queue. */
@@ -1137,6 +1134,10 @@ struct mlx5_dev_ctx_shared {
 	uint32_t tunnel_header_0_1:1; /* tunnel_header_0_1 is supported. */
 	uint32_t misc5_cap:1; /* misc5 matcher parameter is supported. */
 	uint32_t reclaim_mode:1; /* Reclaim memory. */
+	uint32_t dr_drop_action_en:1; /* Use DR drop action. */
+	uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */
+	uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */
+	uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
@@ -1203,6 +1204,9 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_aso_ct_pools_mng *ct_mng;
 	/* Management data for ASO connection tracking. */
 	struct mlx5_lb_ctx self_lb; /* QP to enable self loopback for Devx. */
+	unsigned int flow_max_priority;
+	enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
+	/* Availibility of mreg_c's. */
 	struct mlx5_dev_shared_port port[]; /* per device port data array. */
 };
 
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c914a7120c..3ee03a207e 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -891,7 +891,7 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "invalid tag id");
-		if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
+		if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "unsupported tag id");
@@ -901,21 +901,21 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 		 * If the available index REG_C_y >= REG_C_x, skip the
 		 * color register.
 		 */
-		if (skip_mtr_reg && config->flow_mreg_c
+		if (skip_mtr_reg && priv->sh->flow_mreg_c
 		    [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 			if (id >= (uint32_t)(REG_C_7 - start_reg))
 				return rte_flow_error_set(error, EINVAL,
 						       RTE_FLOW_ERROR_TYPE_ITEM,
 							NULL, "invalid tag id");
-			if (config->flow_mreg_c
+			if (priv->sh->flow_mreg_c
 			    [id + 1 + start_reg - REG_C_0] != REG_NON)
-				return config->flow_mreg_c
+				return priv->sh->flow_mreg_c
 					       [id + 1 + start_reg - REG_C_0];
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "unsupported tag id");
 		}
-		return config->flow_mreg_c[id + start_reg - REG_C_0];
+		return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
 	}
 	MLX5_ASSERT(false);
 	return rte_flow_error_set(error, EINVAL,
@@ -936,7 +936,6 @@ bool
 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_config *config = &priv->config;
 
 	/*
 	 * Having available reg_c can be regarded inclusively as supporting
@@ -946,7 +945,7 @@ mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 	 * - reg_c's are preserved across different domain (FDB and NIC) on
 	 *   packet loopback by flow lookup miss.
 	 */
-	return config->flow_mreg_c[2] != REG_NON;
+	return priv->sh->flow_mreg_c[2] != REG_NON;
 }
 
 /**
@@ -967,7 +966,7 @@ mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	if (!attr->group && !attr->transfer)
-		return priv->config.flow_prio - 2;
+		return priv->sh->flow_max_priority - 2;
 	return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 }
 
@@ -993,7 +992,7 @@ mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 
 	if (!attr->group && !attr->transfer) {
 		if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
-			priority = priv->config.flow_prio - 1;
+			priority = priv->sh->flow_max_priority - 1;
 		return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 	}
 	if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
@@ -1849,7 +1848,7 @@ mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
 			      struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t priority_max = priv->config.flow_prio - 1;
+	uint32_t priority_max = priv->sh->flow_max_priority - 1;
 
 	if (attributes->group)
 		return rte_flow_error_set(error, ENOTSUP,
@@ -7999,13 +7998,12 @@ int
 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_config *config = &priv->config;
 	enum modify_reg idx;
 	int n = 0;
 
 	/* reg_c[0] and reg_c[1] are reserved. */
-	config->flow_mreg_c[n++] = REG_C_0;
-	config->flow_mreg_c[n++] = REG_C_1;
+	priv->sh->flow_mreg_c[n++] = REG_C_0;
+	priv->sh->flow_mreg_c[n++] = REG_C_1;
 	/* Discover availability of other reg_c's. */
 	for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
 		struct rte_flow_attr attr = {
@@ -8041,7 +8039,7 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		struct rte_flow *flow;
 		struct rte_flow_error error;
 
-		if (!config->dv_flow_en)
+		if (!priv->config.dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
 		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
@@ -8050,11 +8048,12 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 				      flow_idx);
 		if (!flow)
 			continue;
-		config->flow_mreg_c[n++] = idx;
+		priv->sh->flow_mreg_c[n++] = idx;
 		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
 	}
 	for (; n < MLX5_MREG_C_NUM; ++n)
-		config->flow_mreg_c[n] = REG_NON;
+		priv->sh->flow_mreg_c[n] = REG_NON;
+	priv->sh->metadata_regc_check_flag = 1;
 	return 0;
 }
 
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index b93fd4d2c9..9dbfaf3c19 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -137,7 +137,7 @@ mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 	uint32_t res = 0;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	switch (priv->config.flow_prio) {
+	switch (priv->sh->flow_max_priority) {
 	case RTE_DIM(priority_map_3):
 		res = priority_map_3[priority][subpriority];
 		break;
@@ -1733,7 +1733,7 @@ flow_verbs_translate(struct rte_eth_dev *dev,
 	MLX5_ASSERT(wks);
 	rss_desc = &wks->rss_desc;
 	if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
-		priority = priv->config.flow_prio - 1;
+		priority = priv->sh->flow_max_priority - 1;
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 		int ret;
 
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 26fa927039..4087d7ca0f 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -636,7 +636,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			goto error;
 	}
 	/* No supported flow priority number detection. */
-	priv->config.flow_prio = -1;
+	priv->sh->flow_max_priority = -1;
 	if (!priv->config.dv_esw_en &&
 	    priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
 		DRV_LOG(WARNING, "metadata mode %u is not supported "
@@ -657,10 +657,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
-	err = mlx5_flow_discover_mreg_c(eth_dev);
-	if (err < 0) {
-		err = -err;
-		goto error;
+	if (!priv->sh->metadata_regc_check_flag) {
+		err = mlx5_flow_discover_mreg_c(eth_dev);
+		if (err < 0) {
+			err = -err;
+			goto error;
+		}
 	}
 	if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
 		DRV_LOG(DEBUG,
-- 
2.18.1

[dpdk-dev] [PATCH v2] net/mlx5: optimize the device spawn time with representors

[Jiawei Wang]

During the device spawn process, mlx5 PMD queried the available flow
priorities by calling mlx5_flow_discover_priorities, queried
if the DR drop action was supported on the root table by calling
the mlx5_flow_discover_dr_action_support routine, and queried the
availability of metadata register C by calling mlx5_flow_discover_mreg_c.

These functions created the test flows to get the supported fields, and
at the end destroyed the test flows. The test flows in the first two
functions was created on the root table.
If the device was spawned with multiple representors, these test flows
were created and destroyed on each representor as well. The above
operations took a significant amount of init time during the device spawn.

This patch optimizes the device discover functions, if there is
the device with multiple representors (VF/SF) being spawned,
the priority and drop action and metadata register support check can be
done only ones and check results can be shared for all representors.

Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
---
v2: Fix the CI warning

---
 drivers/net/mlx5/linux/mlx5_os.c   | 33 +++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.h            | 10 ++++++---
 drivers/net/mlx5/mlx5_flow.c       | 31 ++++++++++++++--------------
 drivers/net/mlx5/mlx5_flow_verbs.c |  4 ++--
 drivers/net/mlx5/windows/mlx5_os.c | 12 ++++++-----
 5 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 3746057673..8a00f15647 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -805,10 +805,15 @@ mlx5_flow_drop_action_config(struct rte_eth_dev *dev __rte_unused)
 	 * DR supports drop action placeholder when it is supported;
 	 * otherwise, use the queue drop action.
 	 */
-	if (mlx5_flow_discover_dr_action_support(dev))
-		priv->root_drop_action = priv->drop_queue.hrxq->action;
-	else
+	if (!priv->sh->drop_action_check_flag) {
+		if (!mlx5_flow_discover_dr_action_support(dev))
+			priv->sh->dr_drop_action_en = 1;
+		priv->sh->drop_action_check_flag = 1;
+	}
+	if (priv->sh->dr_drop_action_en)
 		priv->root_drop_action = priv->sh->dr_drop_action;
+	else
+		priv->root_drop_action = priv->drop_queue.hrxq->action;
 #endif
 }
 
@@ -1830,13 +1835,19 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	priv->drop_queue.hrxq = mlx5_drop_action_create(eth_dev);
 	if (!priv->drop_queue.hrxq)
 		goto error;
-	/* Supported Verbs flow priority number detection. */
-	err = mlx5_flow_discover_priorities(eth_dev);
+	/* Port representor shares the same max prioirity with pf port. */
+	if (!priv->sh->flow_priority_check_flag) {
+		/* Supported Verbs flow priority number detection. */
+		err = mlx5_flow_discover_priorities(eth_dev);
+		priv->sh->flow_max_priority = err;
+		priv->sh->flow_priority_check_flag = 1;
+	} else {
+		err = priv->sh->flow_max_priority;
+	}
 	if (err < 0) {
 		err = -err;
 		goto error;
 	}
-	priv->config.flow_prio = err;
 	if (!priv->config.dv_esw_en &&
 	    priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
 		DRV_LOG(WARNING, "metadata mode %u is not supported "
@@ -1862,10 +1873,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
-	err = mlx5_flow_discover_mreg_c(eth_dev);
-	if (err < 0) {
-		err = -err;
-		goto error;
+	if (!priv->sh->metadata_regc_check_flag) {
+		err = mlx5_flow_discover_mreg_c(eth_dev);
+		if (err < 0) {
+			err = -err;
+			goto error;
+		}
 	}
 	if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
 		DRV_LOG(DEBUG,
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3581414b78..63ec2ce973 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -283,9 +283,6 @@ struct mlx5_dev_config {
 	} mprq; /* Configurations for Multi-Packet RQ. */
 	int mps; /* Multi-packet send supported mode. */
 	int dbnc; /* Skip doorbell register write barrier. */
-	unsigned int flow_prio; /* Number of flow priorities. */
-	enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
-	/* Availibility of mreg_c's. */
 	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
 	unsigned int ind_table_max_size; /* Maximum indirection table size. */
 	unsigned int max_dump_files_num; /* Maximum dump files per queue. */
@@ -1137,6 +1134,10 @@ struct mlx5_dev_ctx_shared {
 	uint32_t tunnel_header_0_1:1; /* tunnel_header_0_1 is supported. */
 	uint32_t misc5_cap:1; /* misc5 matcher parameter is supported. */
 	uint32_t reclaim_mode:1; /* Reclaim memory. */
+	uint32_t dr_drop_action_en:1; /* Use DR drop action. */
+	uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */
+	uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */
+	uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
@@ -1203,6 +1204,9 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_aso_ct_pools_mng *ct_mng;
 	/* Management data for ASO connection tracking. */
 	struct mlx5_lb_ctx self_lb; /* QP to enable self loopback for Devx. */
+	unsigned int flow_max_priority;
+	enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
+	/* Availability of mreg_c's. */
 	struct mlx5_dev_shared_port port[]; /* per device port data array. */
 };
 
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c914a7120c..3ee03a207e 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -891,7 +891,7 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "invalid tag id");
-		if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
+		if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "unsupported tag id");
@@ -901,21 +901,21 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 		 * If the available index REG_C_y >= REG_C_x, skip the
 		 * color register.
 		 */
-		if (skip_mtr_reg && config->flow_mreg_c
+		if (skip_mtr_reg && priv->sh->flow_mreg_c
 		    [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 			if (id >= (uint32_t)(REG_C_7 - start_reg))
 				return rte_flow_error_set(error, EINVAL,
 						       RTE_FLOW_ERROR_TYPE_ITEM,
 							NULL, "invalid tag id");
-			if (config->flow_mreg_c
+			if (priv->sh->flow_mreg_c
 			    [id + 1 + start_reg - REG_C_0] != REG_NON)
-				return config->flow_mreg_c
+				return priv->sh->flow_mreg_c
 					       [id + 1 + start_reg - REG_C_0];
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "unsupported tag id");
 		}
-		return config->flow_mreg_c[id + start_reg - REG_C_0];
+		return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
 	}
 	MLX5_ASSERT(false);
 	return rte_flow_error_set(error, EINVAL,
@@ -936,7 +936,6 @@ bool
 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_config *config = &priv->config;
 
 	/*
 	 * Having available reg_c can be regarded inclusively as supporting
@@ -946,7 +945,7 @@ mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 	 * - reg_c's are preserved across different domain (FDB and NIC) on
 	 *   packet loopback by flow lookup miss.
 	 */
-	return config->flow_mreg_c[2] != REG_NON;
+	return priv->sh->flow_mreg_c[2] != REG_NON;
 }
 
 /**
@@ -967,7 +966,7 @@ mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	if (!attr->group && !attr->transfer)
-		return priv->config.flow_prio - 2;
+		return priv->sh->flow_max_priority - 2;
 	return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 }
 
@@ -993,7 +992,7 @@ mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 
 	if (!attr->group && !attr->transfer) {
 		if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
-			priority = priv->config.flow_prio - 1;
+			priority = priv->sh->flow_max_priority - 1;
 		return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 	}
 	if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
@@ -1849,7 +1848,7 @@ mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
 			      struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t priority_max = priv->config.flow_prio - 1;
+	uint32_t priority_max = priv->sh->flow_max_priority - 1;
 
 	if (attributes->group)
 		return rte_flow_error_set(error, ENOTSUP,
@@ -7999,13 +7998,12 @@ int
 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_config *config = &priv->config;
 	enum modify_reg idx;
 	int n = 0;
 
 	/* reg_c[0] and reg_c[1] are reserved. */
-	config->flow_mreg_c[n++] = REG_C_0;
-	config->flow_mreg_c[n++] = REG_C_1;
+	priv->sh->flow_mreg_c[n++] = REG_C_0;
+	priv->sh->flow_mreg_c[n++] = REG_C_1;
 	/* Discover availability of other reg_c's. */
 	for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
 		struct rte_flow_attr attr = {
@@ -8041,7 +8039,7 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		struct rte_flow *flow;
 		struct rte_flow_error error;
 
-		if (!config->dv_flow_en)
+		if (!priv->config.dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
 		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
@@ -8050,11 +8048,12 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 				      flow_idx);
 		if (!flow)
 			continue;
-		config->flow_mreg_c[n++] = idx;
+		priv->sh->flow_mreg_c[n++] = idx;
 		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
 	}
 	for (; n < MLX5_MREG_C_NUM; ++n)
-		config->flow_mreg_c[n] = REG_NON;
+		priv->sh->flow_mreg_c[n] = REG_NON;
+	priv->sh->metadata_regc_check_flag = 1;
 	return 0;
 }
 
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index b93fd4d2c9..9dbfaf3c19 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -137,7 +137,7 @@ mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 	uint32_t res = 0;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	switch (priv->config.flow_prio) {
+	switch (priv->sh->flow_max_priority) {
 	case RTE_DIM(priority_map_3):
 		res = priority_map_3[priority][subpriority];
 		break;
@@ -1733,7 +1733,7 @@ flow_verbs_translate(struct rte_eth_dev *dev,
 	MLX5_ASSERT(wks);
 	rss_desc = &wks->rss_desc;
 	if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
-		priority = priv->config.flow_prio - 1;
+		priority = priv->sh->flow_max_priority - 1;
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 		int ret;
 
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 26fa927039..4087d7ca0f 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -636,7 +636,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			goto error;
 	}
 	/* No supported flow priority number detection. */
-	priv->config.flow_prio = -1;
+	priv->sh->flow_max_priority = -1;
 	if (!priv->config.dv_esw_en &&
 	    priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
 		DRV_LOG(WARNING, "metadata mode %u is not supported "
@@ -657,10 +657,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
-	err = mlx5_flow_discover_mreg_c(eth_dev);
-	if (err < 0) {
-		err = -err;
-		goto error;
+	if (!priv->sh->metadata_regc_check_flag) {
+		err = mlx5_flow_discover_mreg_c(eth_dev);
+		if (err < 0) {
+			err = -err;
+			goto error;
+		}
 	}
 	if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
 		DRV_LOG(DEBUG,
-- 
2.18.1

Re: [dpdk-dev] [PATCH v2] net/mlx5: optimize the device spawn time with representors

[Raslan Darawsheh]

Hi,
> -----Original Message-----
> From: Jiawei(Jonny) Wang <jiaweiw@nvidia.com>
> Sent: Thursday, September 30, 2021 3:01 PM
> To: Slava Ovsiienko <viacheslavo@nvidia.com>; Matan Azrad
> <matan@nvidia.com>; Ori Kam <orika@nvidia.com>; NBU-Contact-Thomas
> Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Raslan Darawsheh <rasland@nvidia.com>
> Subject: [PATCH v2] net/mlx5: optimize the device spawn time with
> representors
> 
> During the device spawn process, mlx5 PMD queried the available flow
> priorities by calling mlx5_flow_discover_priorities, queried
> if the DR drop action was supported on the root table by calling
> the mlx5_flow_discover_dr_action_support routine, and queried the
> availability of metadata register C by calling mlx5_flow_discover_mreg_c.
> 
> These functions created the test flows to get the supported fields, and
> at the end destroyed the test flows. The test flows in the first two
> functions was created on the root table.
> If the device was spawned with multiple representors, these test flows
> were created and destroyed on each representor as well. The above
> operations took a significant amount of init time during the device spawn.
> 
> This patch optimizes the device discover functions, if there is
> the device with multiple representors (VF/SF) being spawned,
> the priority and drop action and metadata register support check can be
> done only ones and check results can be shared for all representors.
> 
> Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
> Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
> ---
> v2: Fix the CI warning
> 
> ---
>  drivers/net/mlx5/linux/mlx5_os.c   | 33 +++++++++++++++++++++---------
>  drivers/net/mlx5/mlx5.h            | 10 ++++++---
>  drivers/net/mlx5/mlx5_flow.c       | 31 ++++++++++++++--------------
>  drivers/net/mlx5/mlx5_flow_verbs.c |  4 ++--
>  drivers/net/mlx5/windows/mlx5_os.c | 12 ++++++-----
>  5 files changed, 54 insertions(+), 36 deletions(-)
> 

Patch rebased and applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh

[dpdk-dev] [PATCH v3] net/mlx5: optimize the device spawn time with representors

[Jiawei Wang]

During the device spawn process, mlx5 PMD queried the available flow
priorities by calling mlx5_flow_discover_priorities, queried
if the DR drop action was supported on the root table by calling
the mlx5_flow_discover_dr_action_support routine, and queried the
availability of metadata register C by calling mlx5_flow_discover_mreg_c.

These functions created the test flows to get the supported fields, and
at the end destroyed the test flows. The test flows in the first two
functions was created on the root table.
If the device was spawned with multiple representors, these test flows
were created and destroyed on each representor as well. The above
operations took a significant amount of init time during the device spawn.

This patch optimizes the device discover functions, if there is
the device with multiple representors (VF/SF) being spawned,
the priority and drop action and metadata register support check can be
done only ones and check results can be shared for all representors.

Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
---
v3: Rebase
v2: Fix the CI warning

---
 drivers/net/mlx5/linux/mlx5_os.c   | 33 +++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.h            | 10 ++++++---
 drivers/net/mlx5/mlx5_flow.c       | 33 +++++++++++++++---------------
 drivers/net/mlx5/mlx5_flow_verbs.c |  4 ++--
 drivers/net/mlx5/windows/mlx5_os.c | 12 ++++++-----
 5 files changed, 55 insertions(+), 37 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 3deae861d5..f31f1e96c6 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -704,10 +704,15 @@ mlx5_flow_drop_action_config(struct rte_eth_dev *dev __rte_unused)
 	 * DR supports drop action placeholder when it is supported;
 	 * otherwise, use the queue drop action.
 	 */
-	if (mlx5_flow_discover_dr_action_support(dev))
-		priv->root_drop_action = priv->drop_queue.hrxq->action;
-	else
+	if (!priv->sh->drop_action_check_flag) {
+		if (!mlx5_flow_discover_dr_action_support(dev))
+			priv->sh->dr_drop_action_en = 1;
+		priv->sh->drop_action_check_flag = 1;
+	}
+	if (priv->sh->dr_drop_action_en)
 		priv->root_drop_action = priv->sh->dr_drop_action;
+	else
+		priv->root_drop_action = priv->drop_queue.hrxq->action;
 #endif
 }
 
@@ -1720,13 +1725,19 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	priv->drop_queue.hrxq = mlx5_drop_action_create(eth_dev);
 	if (!priv->drop_queue.hrxq)
 		goto error;
-	/* Supported Verbs flow priority number detection. */
-	err = mlx5_flow_discover_priorities(eth_dev);
+	/* Port representor shares the same max prioirity with pf port. */
+	if (!priv->sh->flow_priority_check_flag) {
+		/* Supported Verbs flow priority number detection. */
+		err = mlx5_flow_discover_priorities(eth_dev);
+		priv->sh->flow_max_priority = err;
+		priv->sh->flow_priority_check_flag = 1;
+	} else {
+		err = priv->sh->flow_max_priority;
+	}
 	if (err < 0) {
 		err = -err;
 		goto error;
 	}
-	priv->config.flow_prio = err;
 	if (!priv->config.dv_esw_en &&
 	    priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
 		DRV_LOG(WARNING, "metadata mode %u is not supported "
@@ -1752,10 +1763,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
-	err = mlx5_flow_discover_mreg_c(eth_dev);
-	if (err < 0) {
-		err = -err;
-		goto error;
+	if (!priv->sh->metadata_regc_check_flag) {
+		err = mlx5_flow_discover_mreg_c(eth_dev);
+		if (err < 0) {
+			err = -err;
+			goto error;
+		}
 	}
 	if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
 		DRV_LOG(DEBUG,
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 5768b82935..4d16c784c3 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -270,9 +270,6 @@ struct mlx5_dev_config {
 		/* Rx queue count threshold to enable MPRQ. */
 	} mprq; /* Configurations for Multi-Packet RQ. */
 	int mps; /* Multi-packet send supported mode. */
-	unsigned int flow_prio; /* Number of flow priorities. */
-	enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
-	/* Availibility of mreg_c's. */
 	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
 	unsigned int ind_table_max_size; /* Maximum indirection table size. */
 	unsigned int max_dump_files_num; /* Maximum dump files per queue. */
@@ -1120,6 +1117,10 @@ struct mlx5_dev_ctx_shared {
 	uint32_t tunnel_header_0_1:1; /* tunnel_header_0_1 is supported. */
 	uint32_t misc5_cap:1; /* misc5 matcher parameter is supported. */
 	uint32_t reclaim_mode:1; /* Reclaim memory. */
+	uint32_t dr_drop_action_en:1; /* Use DR drop action. */
+	uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */
+	uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */
+	uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	struct mlx5_common_device *cdev; /* Backend mlx5 device. */
@@ -1180,6 +1181,9 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_aso_ct_pools_mng *ct_mng;
 	/* Management data for ASO connection tracking. */
 	struct mlx5_lb_ctx self_lb; /* QP to enable self loopback for Devx. */
+	unsigned int flow_max_priority;
+	enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
+	/* Availability of mreg_c's. */
 	struct mlx5_dev_shared_port port[]; /* per device port data array. */
 };
 
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 5d19ef1e82..f842ce0802 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -914,7 +914,7 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "invalid tag id");
-		if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
+		if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "unsupported tag id");
@@ -924,21 +924,21 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 		 * If the available index REG_C_y >= REG_C_x, skip the
 		 * color register.
 		 */
-		if (skip_mtr_reg && config->flow_mreg_c
+		if (skip_mtr_reg && priv->sh->flow_mreg_c
 		    [id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
 			if (id >= (uint32_t)(REG_C_7 - start_reg))
 				return rte_flow_error_set(error, EINVAL,
 						       RTE_FLOW_ERROR_TYPE_ITEM,
 							NULL, "invalid tag id");
-			if (config->flow_mreg_c
+			if (priv->sh->flow_mreg_c
 			    [id + 1 + start_reg - REG_C_0] != REG_NON)
-				return config->flow_mreg_c
+				return priv->sh->flow_mreg_c
 					       [id + 1 + start_reg - REG_C_0];
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "unsupported tag id");
 		}
-		return config->flow_mreg_c[id + start_reg - REG_C_0];
+		return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
 	}
 	MLX5_ASSERT(false);
 	return rte_flow_error_set(error, EINVAL,
@@ -959,7 +959,6 @@ bool
 mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_config *config = &priv->config;
 
 	/*
 	 * Having available reg_c can be regarded inclusively as supporting
@@ -969,7 +968,7 @@ mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
 	 * - reg_c's are preserved across different domain (FDB and NIC) on
 	 *   packet loopback by flow lookup miss.
 	 */
-	return config->flow_mreg_c[2] != REG_NON;
+	return priv->sh->flow_mreg_c[2] != REG_NON;
 }
 
 /**
@@ -990,7 +989,7 @@ mlx5_get_lowest_priority(struct rte_eth_dev *dev,
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	if (!attr->group && !attr->transfer)
-		return priv->config.flow_prio - 2;
+		return priv->sh->flow_max_priority - 2;
 	return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
 }
 
@@ -1018,11 +1017,11 @@ mlx5_get_matcher_priority(struct rte_eth_dev *dev,
 
 	if (!attr->group && !attr->transfer) {
 		if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
-			priority = priv->config.flow_prio - 1;
+			priority = priv->sh->flow_max_priority - 1;
 		return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
 	} else if (!external && attr->transfer && attr->group == 0 &&
 		   attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) {
-		return (priv->config.flow_prio - 1) * 3;
+		return (priv->sh->flow_max_priority - 1) * 3;
 	}
 	if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
 		priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
@@ -1877,7 +1876,7 @@ mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
 			      struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t priority_max = priv->config.flow_prio - 1;
+	uint32_t priority_max = priv->sh->flow_max_priority - 1;
 
 	if (attributes->group)
 		return rte_flow_error_set(error, ENOTSUP,
@@ -8112,13 +8111,12 @@ int
 mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_config *config = &priv->config;
 	enum modify_reg idx;
 	int n = 0;
 
 	/* reg_c[0] and reg_c[1] are reserved. */
-	config->flow_mreg_c[n++] = REG_C_0;
-	config->flow_mreg_c[n++] = REG_C_1;
+	priv->sh->flow_mreg_c[n++] = REG_C_0;
+	priv->sh->flow_mreg_c[n++] = REG_C_1;
 	/* Discover availability of other reg_c's. */
 	for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
 		struct rte_flow_attr attr = {
@@ -8154,7 +8152,7 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		struct rte_flow *flow;
 		struct rte_flow_error error;
 
-		if (!config->dv_flow_en)
+		if (!priv->config.dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
 		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
@@ -8163,11 +8161,12 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 				      flow_idx);
 		if (!flow)
 			continue;
-		config->flow_mreg_c[n++] = idx;
+		priv->sh->flow_mreg_c[n++] = idx;
 		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
 	}
 	for (; n < MLX5_MREG_C_NUM; ++n)
-		config->flow_mreg_c[n] = REG_NON;
+		priv->sh->flow_mreg_c[n] = REG_NON;
+	priv->sh->metadata_regc_check_flag = 1;
 	return 0;
 }
 
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 176d867202..0a89a136a2 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -145,7 +145,7 @@ mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
 	uint32_t res = 0;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	switch (priv->config.flow_prio) {
+	switch (priv->sh->flow_max_priority) {
 	case RTE_DIM(priority_map_3):
 		res = priority_map_3[priority][subpriority];
 		break;
@@ -1723,7 +1723,7 @@ flow_verbs_translate(struct rte_eth_dev *dev,
 	MLX5_ASSERT(wks);
 	rss_desc = &wks->rss_desc;
 	if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
-		priority = priv->config.flow_prio - 1;
+		priority = priv->sh->flow_max_priority - 1;
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 		int ret;
 
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 5e88453b03..dec4b923d0 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -605,7 +605,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			goto error;
 	}
 	/* No supported flow priority number detection. */
-	priv->config.flow_prio = -1;
+	priv->sh->flow_max_priority = -1;
 	if (!priv->config.dv_esw_en &&
 	    priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
 		DRV_LOG(WARNING, "metadata mode %u is not supported "
@@ -626,10 +626,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
-	err = mlx5_flow_discover_mreg_c(eth_dev);
-	if (err < 0) {
-		err = -err;
-		goto error;
+	if (!priv->sh->metadata_regc_check_flag) {
+		err = mlx5_flow_discover_mreg_c(eth_dev);
+		if (err < 0) {
+			err = -err;
+			goto error;
+		}
 	}
 	if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
 		DRV_LOG(DEBUG,
-- 
2.18.1

Re: [dpdk-dev] [PATCH v3] net/mlx5: optimize the device spawn time with representors

[Raslan Darawsheh]

Hi,

> -----Original Message-----
> From: Jiawei(Jonny) Wang <jiaweiw@nvidia.com>
> Sent: Wednesday, October 27, 2021 1:35 PM
> To: Slava Ovsiienko <viacheslavo@nvidia.com>; Matan Azrad
> <matan@nvidia.com>; Ori Kam <orika@nvidia.com>; NBU-Contact-Thomas
> Monjalon <thomas@monjalon.net>
> Cc: dev@dpdk.org; Raslan Darawsheh <rasland@nvidia.com>
> Subject: [PATCH v3] net/mlx5: optimize the device spawn time with
> representors
> 
> During the device spawn process, mlx5 PMD queried the available flow
> priorities by calling mlx5_flow_discover_priorities, queried
> if the DR drop action was supported on the root table by calling
> the mlx5_flow_discover_dr_action_support routine, and queried the
> availability of metadata register C by calling mlx5_flow_discover_mreg_c.
> 
> These functions created the test flows to get the supported fields, and
> at the end destroyed the test flows. The test flows in the first two
> functions was created on the root table.
> If the device was spawned with multiple representors, these test flows
> were created and destroyed on each representor as well. The above
> operations took a significant amount of init time during the device spawn.
> 
> This patch optimizes the device discover functions, if there is
> the device with multiple representors (VF/SF) being spawned,
> the priority and drop action and metadata register support check can be
> done only ones and check results can be shared for all representors.
> 
> Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
> Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
> ---
> v3: Rebase
> v2: Fix the CI warning
> 

Removed v2 from next-net-mlx,
And applied v3 to next-net-mlx,

Kindest regards,
Raslan Darawsheh