From: Jiawei Wang <jiaweiw@nvidia.com>
To: <viacheslavo@nvidia.com>, <matan@nvidia.com>, <orika@nvidia.com>,
<thomas@monjalon.net>
Cc: <dev@dpdk.org>, <rasland@nvidia.com>
Subject: [dpdk-dev] [PATCH v3] net/mlx5: optimize the device spawn time with representors
Date: Wed, 27 Oct 2021 13:35:10 +0300 [thread overview]
Message-ID: <20211027103510.919-1-jiaweiw@nvidia.com> (raw)
In-Reply-To: <20210930115217.35210-1-jiaweiw@nvidia.com>
During the device spawn process, mlx5 PMD queried the available flow
priorities by calling mlx5_flow_discover_priorities, queried
if the DR drop action was supported on the root table by calling
the mlx5_flow_discover_dr_action_support routine, and queried the
availability of metadata register C by calling mlx5_flow_discover_mreg_c.
These functions created the test flows to get the supported fields, and
at the end destroyed the test flows. The test flows in the first two
functions was created on the root table.
If the device was spawned with multiple representors, these test flows
were created and destroyed on each representor as well. The above
operations took a significant amount of init time during the device spawn.
This patch optimizes the device discover functions, if there is
the device with multiple representors (VF/SF) being spawned,
the priority and drop action and metadata register support check can be
done only ones and check results can be shared for all representors.
Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
---
v3: Rebase
v2: Fix the CI warning
---
drivers/net/mlx5/linux/mlx5_os.c | 33 +++++++++++++++++++++---------
drivers/net/mlx5/mlx5.h | 10 ++++++---
drivers/net/mlx5/mlx5_flow.c | 33 +++++++++++++++---------------
drivers/net/mlx5/mlx5_flow_verbs.c | 4 ++--
drivers/net/mlx5/windows/mlx5_os.c | 12 ++++++-----
5 files changed, 55 insertions(+), 37 deletions(-)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 3deae861d5..f31f1e96c6 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -704,10 +704,15 @@ mlx5_flow_drop_action_config(struct rte_eth_dev *dev __rte_unused)
* DR supports drop action placeholder when it is supported;
* otherwise, use the queue drop action.
*/
- if (mlx5_flow_discover_dr_action_support(dev))
- priv->root_drop_action = priv->drop_queue.hrxq->action;
- else
+ if (!priv->sh->drop_action_check_flag) {
+ if (!mlx5_flow_discover_dr_action_support(dev))
+ priv->sh->dr_drop_action_en = 1;
+ priv->sh->drop_action_check_flag = 1;
+ }
+ if (priv->sh->dr_drop_action_en)
priv->root_drop_action = priv->sh->dr_drop_action;
+ else
+ priv->root_drop_action = priv->drop_queue.hrxq->action;
#endif
}
@@ -1720,13 +1725,19 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
priv->drop_queue.hrxq = mlx5_drop_action_create(eth_dev);
if (!priv->drop_queue.hrxq)
goto error;
- /* Supported Verbs flow priority number detection. */
- err = mlx5_flow_discover_priorities(eth_dev);
+ /* Port representor shares the same max prioirity with pf port. */
+ if (!priv->sh->flow_priority_check_flag) {
+ /* Supported Verbs flow priority number detection. */
+ err = mlx5_flow_discover_priorities(eth_dev);
+ priv->sh->flow_max_priority = err;
+ priv->sh->flow_priority_check_flag = 1;
+ } else {
+ err = priv->sh->flow_max_priority;
+ }
if (err < 0) {
err = -err;
goto error;
}
- priv->config.flow_prio = err;
if (!priv->config.dv_esw_en &&
priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
DRV_LOG(WARNING, "metadata mode %u is not supported "
@@ -1752,10 +1763,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
goto error;
rte_rwlock_init(&priv->ind_tbls_lock);
/* Query availability of metadata reg_c's. */
- err = mlx5_flow_discover_mreg_c(eth_dev);
- if (err < 0) {
- err = -err;
- goto error;
+ if (!priv->sh->metadata_regc_check_flag) {
+ err = mlx5_flow_discover_mreg_c(eth_dev);
+ if (err < 0) {
+ err = -err;
+ goto error;
+ }
}
if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
DRV_LOG(DEBUG,
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 5768b82935..4d16c784c3 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -270,9 +270,6 @@ struct mlx5_dev_config {
/* Rx queue count threshold to enable MPRQ. */
} mprq; /* Configurations for Multi-Packet RQ. */
int mps; /* Multi-packet send supported mode. */
- unsigned int flow_prio; /* Number of flow priorities. */
- enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
- /* Availibility of mreg_c's. */
unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
unsigned int ind_table_max_size; /* Maximum indirection table size. */
unsigned int max_dump_files_num; /* Maximum dump files per queue. */
@@ -1120,6 +1117,10 @@ struct mlx5_dev_ctx_shared {
uint32_t tunnel_header_0_1:1; /* tunnel_header_0_1 is supported. */
uint32_t misc5_cap:1; /* misc5 matcher parameter is supported. */
uint32_t reclaim_mode:1; /* Reclaim memory. */
+ uint32_t dr_drop_action_en:1; /* Use DR drop action. */
+ uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */
+ uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */
+ uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */
uint32_t max_port; /* Maximal IB device port index. */
struct mlx5_bond_info bond; /* Bonding information. */
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
@@ -1180,6 +1181,9 @@ struct mlx5_dev_ctx_shared {
struct mlx5_aso_ct_pools_mng *ct_mng;
/* Management data for ASO connection tracking. */
struct mlx5_lb_ctx self_lb; /* QP to enable self loopback for Devx. */
+ unsigned int flow_max_priority;
+ enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
+ /* Availability of mreg_c's. */
struct mlx5_dev_shared_port port[]; /* per device port data array. */
};
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 5d19ef1e82..f842ce0802 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -914,7 +914,7 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM,
NULL, "invalid tag id");
- if (config->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
+ if (priv->sh->flow_mreg_c[id + start_reg - REG_C_0] == REG_NON)
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ITEM,
NULL, "unsupported tag id");
@@ -924,21 +924,21 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
* If the available index REG_C_y >= REG_C_x, skip the
* color register.
*/
- if (skip_mtr_reg && config->flow_mreg_c
+ if (skip_mtr_reg && priv->sh->flow_mreg_c
[id + start_reg - REG_C_0] >= priv->mtr_color_reg) {
if (id >= (uint32_t)(REG_C_7 - start_reg))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM,
NULL, "invalid tag id");
- if (config->flow_mreg_c
+ if (priv->sh->flow_mreg_c
[id + 1 + start_reg - REG_C_0] != REG_NON)
- return config->flow_mreg_c
+ return priv->sh->flow_mreg_c
[id + 1 + start_reg - REG_C_0];
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ITEM,
NULL, "unsupported tag id");
}
- return config->flow_mreg_c[id + start_reg - REG_C_0];
+ return priv->sh->flow_mreg_c[id + start_reg - REG_C_0];
}
MLX5_ASSERT(false);
return rte_flow_error_set(error, EINVAL,
@@ -959,7 +959,6 @@ bool
mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_dev_config *config = &priv->config;
/*
* Having available reg_c can be regarded inclusively as supporting
@@ -969,7 +968,7 @@ mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev)
* - reg_c's are preserved across different domain (FDB and NIC) on
* packet loopback by flow lookup miss.
*/
- return config->flow_mreg_c[2] != REG_NON;
+ return priv->sh->flow_mreg_c[2] != REG_NON;
}
/**
@@ -990,7 +989,7 @@ mlx5_get_lowest_priority(struct rte_eth_dev *dev,
struct mlx5_priv *priv = dev->data->dev_private;
if (!attr->group && !attr->transfer)
- return priv->config.flow_prio - 2;
+ return priv->sh->flow_max_priority - 2;
return MLX5_NON_ROOT_FLOW_MAX_PRIO - 1;
}
@@ -1018,11 +1017,11 @@ mlx5_get_matcher_priority(struct rte_eth_dev *dev,
if (!attr->group && !attr->transfer) {
if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
- priority = priv->config.flow_prio - 1;
+ priority = priv->sh->flow_max_priority - 1;
return mlx5_os_flow_adjust_priority(dev, priority, subpriority);
} else if (!external && attr->transfer && attr->group == 0 &&
attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) {
- return (priv->config.flow_prio - 1) * 3;
+ return (priv->sh->flow_max_priority - 1) * 3;
}
if (attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
priority = MLX5_NON_ROOT_FLOW_MAX_PRIO;
@@ -1877,7 +1876,7 @@ mlx5_flow_validate_attributes(struct rte_eth_dev *dev,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
- uint32_t priority_max = priv->config.flow_prio - 1;
+ uint32_t priority_max = priv->sh->flow_max_priority - 1;
if (attributes->group)
return rte_flow_error_set(error, ENOTSUP,
@@ -8112,13 +8111,12 @@ int
mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_dev_config *config = &priv->config;
enum modify_reg idx;
int n = 0;
/* reg_c[0] and reg_c[1] are reserved. */
- config->flow_mreg_c[n++] = REG_C_0;
- config->flow_mreg_c[n++] = REG_C_1;
+ priv->sh->flow_mreg_c[n++] = REG_C_0;
+ priv->sh->flow_mreg_c[n++] = REG_C_1;
/* Discover availability of other reg_c's. */
for (idx = REG_C_2; idx <= REG_C_7; ++idx) {
struct rte_flow_attr attr = {
@@ -8154,7 +8152,7 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
struct rte_flow *flow;
struct rte_flow_error error;
- if (!config->dv_flow_en)
+ if (!priv->config.dv_flow_en)
break;
/* Create internal flow, validation skips copy action. */
flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
@@ -8163,11 +8161,12 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
flow_idx);
if (!flow)
continue;
- config->flow_mreg_c[n++] = idx;
+ priv->sh->flow_mreg_c[n++] = idx;
flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
}
for (; n < MLX5_MREG_C_NUM; ++n)
- config->flow_mreg_c[n] = REG_NON;
+ priv->sh->flow_mreg_c[n] = REG_NON;
+ priv->sh->metadata_regc_check_flag = 1;
return 0;
}
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 176d867202..0a89a136a2 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -145,7 +145,7 @@ mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
uint32_t res = 0;
struct mlx5_priv *priv = dev->data->dev_private;
- switch (priv->config.flow_prio) {
+ switch (priv->sh->flow_max_priority) {
case RTE_DIM(priority_map_3):
res = priority_map_3[priority][subpriority];
break;
@@ -1723,7 +1723,7 @@ flow_verbs_translate(struct rte_eth_dev *dev,
MLX5_ASSERT(wks);
rss_desc = &wks->rss_desc;
if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)
- priority = priv->config.flow_prio - 1;
+ priority = priv->sh->flow_max_priority - 1;
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
int ret;
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 5e88453b03..dec4b923d0 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -605,7 +605,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
goto error;
}
/* No supported flow priority number detection. */
- priv->config.flow_prio = -1;
+ priv->sh->flow_max_priority = -1;
if (!priv->config.dv_esw_en &&
priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
DRV_LOG(WARNING, "metadata mode %u is not supported "
@@ -626,10 +626,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
mlx5_hrxq_clone_free_cb);
/* Query availability of metadata reg_c's. */
- err = mlx5_flow_discover_mreg_c(eth_dev);
- if (err < 0) {
- err = -err;
- goto error;
+ if (!priv->sh->metadata_regc_check_flag) {
+ err = mlx5_flow_discover_mreg_c(eth_dev);
+ if (err < 0) {
+ err = -err;
+ goto error;
+ }
}
if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
DRV_LOG(DEBUG,
--
2.18.1
next prev parent reply other threads:[~2021-10-27 10:35 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-09-30 11:52 [dpdk-dev] [PATCH] " Jiawei Wang
2021-09-30 12:00 ` [dpdk-dev] [PATCH v2] " Jiawei Wang
2021-10-26 12:40 ` Raslan Darawsheh
2021-10-27 10:35 ` Jiawei Wang [this message]
2021-10-27 12:06 ` [dpdk-dev] [PATCH v3] " Raslan Darawsheh
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211027103510.919-1-jiaweiw@nvidia.com \
--to=jiaweiw@nvidia.com \
--cc=dev@dpdk.org \
--cc=matan@nvidia.com \
--cc=orika@nvidia.com \
--cc=rasland@nvidia.com \
--cc=thomas@monjalon.net \
--cc=viacheslavo@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).