From: "Minggang Li(Gavin)" <gavinl@nvidia.com>
To: <viacheslavo@nvidia.com>, <matan@nvidia.com>, <orika@nvidia.com>,
<thomas@monjalon.net>, Dariusz Sosnowski <dsosnowski@nvidia.com>,
Bing Zhao <bingz@nvidia.com>, Suanming Mou <suanmingm@nvidia.com>
Cc: <dev@dpdk.org>, <rasland@nvidia.com>, Rongwei Liu <rongweil@nvidia.com>
Subject: [PATCH V1 3/7] net/mlx5: add new devargs to control probe optimization
Date: Wed, 16 Oct 2024 11:38:14 +0300 [thread overview]
Message-ID: <20241016083818.662020-4-gavinl@nvidia.com> (raw)
In-Reply-To: <20241016083818.662020-1-gavinl@nvidia.com>
From: Rongwei Liu <rongweil@nvidia.com>
Add a new devarg probe_opt_en to control probe optimization
in PMD.
By default, the value is 0 and no behavior changed.
Signed-off-by: Rongwei Liu <rongweil@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
---
doc/guides/nics/mlx5.rst | 7 +++++++
drivers/common/mlx5/linux/mlx5_nl.c | 12 ++++++++----
drivers/common/mlx5/mlx5_common.c | 15 +++++++++++++++
drivers/common/mlx5/mlx5_common.h | 2 ++
drivers/net/mlx5/linux/mlx5_ethdev_os.c | 5 ++++-
drivers/net/mlx5/linux/mlx5_os.c | 2 +-
6 files changed, 37 insertions(+), 6 deletions(-)
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 1dccdaad50..b4a4e57cde 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -1436,6 +1436,13 @@ for an additional list of options shared with other mlx5 drivers.
By default, the PMD will set this value to 1.
+- ``probe_opt_en`` parameter [int]
+
+ A non-zero value optimizes the probe process, especially for large scale.
+ PMD will hold the IB device information internally and reuse it.
+
+ By default, the PMD will set this value to 0.
+
- ``lacp_by_user`` parameter [int]
A nonzero value enables the control of LACP traffic by the user application.
diff --git a/drivers/common/mlx5/linux/mlx5_nl.c b/drivers/common/mlx5/linux/mlx5_nl.c
index e98073aafe..745e443f8f 100644
--- a/drivers/common/mlx5/linux/mlx5_nl.c
+++ b/drivers/common/mlx5/linux/mlx5_nl.c
@@ -1148,7 +1148,7 @@ mlx5_nl_ifindex(int nl, const char *name, uint32_t pindex, struct mlx5_dev_info
.flags = 0,
};
- if (!strcmp(name, dev_info->ibname)) {
+ if (dev_info->probe_opt && !strcmp(name, dev_info->ibname)) {
if (dev_info->port_info && pindex <= dev_info->port_num &&
dev_info->port_info[pindex].valid) {
if (!dev_info->port_info[pindex].ifindex)
@@ -1161,7 +1161,7 @@ mlx5_nl_ifindex(int nl, const char *name, uint32_t pindex, struct mlx5_dev_info
ret = mlx5_nl_port_info(nl, pindex, &data);
- if (!strcmp(dev_info->ibname, name)) {
+ if (dev_info->probe_opt && !strcmp(dev_info->ibname, name)) {
if ((!ret || ret == -ENODEV) && dev_info->port_info &&
pindex <= dev_info->port_num) {
if (!ret)
@@ -1201,7 +1201,8 @@ mlx5_nl_port_state(int nl, const char *name, uint32_t pindex, struct mlx5_dev_in
.ibindex = UINT32_MAX,
};
- if (dev_info && !strcmp(name, dev_info->ibname) && dev_info->port_num)
+ if (dev_info && dev_info->probe_opt &&
+ !strcmp(name, dev_info->ibname) && dev_info->port_num)
data.ibindex = dev_info->ibindex;
if (mlx5_nl_port_info(nl, pindex, &data) < 0)
return -rte_errno;
@@ -1244,7 +1245,8 @@ mlx5_nl_portnum(int nl, const char *name, struct mlx5_dev_info *dev_info)
uint32_t sn = MLX5_NL_SN_GENERATE;
int ret, size;
- if (dev_info->port_num && !strcmp(name, dev_info->ibname))
+ if (dev_info->probe_opt && dev_info->port_num &&
+ !strcmp(name, dev_info->ibname))
return dev_info->port_num;
ret = mlx5_nl_send(nl, &req, sn);
@@ -1263,6 +1265,8 @@ mlx5_nl_portnum(int nl, const char *name, struct mlx5_dev_info *dev_info)
rte_errno = EINVAL;
return 0;
}
+ if (!dev_info->probe_opt)
+ return data.portnum;
MLX5_ASSERT(!strlen(dev_info->ibname));
dev_info->port_num = data.portnum;
dev_info->ibindex = data.ibindex;
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index 0aaae91c31..9abae4a374 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -40,6 +40,9 @@ uint8_t haswell_broadwell_cpu;
/* The default memory allocator used in PMD. */
#define MLX5_SYS_MEM_EN "sys_mem_en"
+/* Probe optimization in PMD. */
+#define MLX5_PROBE_OPT "probe_opt_en"
+
/*
* Device parameter to force doorbell register mapping
* to non-cached region eliminating the extra write memory barrier.
@@ -295,6 +298,8 @@ mlx5_common_args_check_handler(const char *key, const char *val, void *opaque)
config->device_fd = tmp;
} else if (strcmp(key, MLX5_PD_HANDLE) == 0) {
config->pd_handle = tmp;
+ } else if (strcmp(key, MLX5_PROBE_OPT) == 0) {
+ config->probe_opt = !!tmp;
}
return 0;
}
@@ -324,6 +329,7 @@ mlx5_common_config_get(struct mlx5_kvargs_ctrl *mkvlist,
MLX5_MR_MEMPOOL_REG_EN,
MLX5_DEVICE_FD,
MLX5_PD_HANDLE,
+ MLX5_PROBE_OPT,
NULL,
};
int ret = 0;
@@ -332,6 +338,7 @@ mlx5_common_config_get(struct mlx5_kvargs_ctrl *mkvlist,
config->mr_ext_memseg_en = 1;
config->mr_mempool_reg_en = 1;
config->sys_mem_en = 0;
+ config->probe_opt = 0;
config->dbnc = MLX5_ARG_UNSET;
config->device_fd = MLX5_ARG_UNSET;
config->pd_handle = MLX5_ARG_UNSET;
@@ -351,6 +358,7 @@ mlx5_common_config_get(struct mlx5_kvargs_ctrl *mkvlist,
DRV_LOG(DEBUG, "mr_ext_memseg_en is %u.", config->mr_ext_memseg_en);
DRV_LOG(DEBUG, "mr_mempool_reg_en is %u.", config->mr_mempool_reg_en);
DRV_LOG(DEBUG, "sys_mem_en is %u.", config->sys_mem_en);
+ DRV_LOG(DEBUG, "probe_opt_en is %u.", config->probe_opt);
DRV_LOG(DEBUG, "Send Queue doorbell mapping parameter is %d.",
config->dbnc);
return ret;
@@ -791,6 +799,7 @@ mlx5_common_dev_create(struct rte_device *eal_dev, uint32_t classes,
if (TAILQ_EMPTY(&devices_list))
rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
mlx5_mr_mem_event_cb, NULL);
+ cdev->dev_info.probe_opt = cdev->config.probe_opt;
exit:
pthread_mutex_lock(&devices_list_lock);
TAILQ_INSERT_HEAD(&devices_list, cdev, next);
@@ -880,6 +889,12 @@ mlx5_common_probe_again_args_validate(struct mlx5_common_device *cdev,
cdev->dev->name);
goto error;
}
+ if (cdev->config.probe_opt != config->probe_opt) {
+ DRV_LOG(ERR, "\"" MLX5_PROBE_OPT"\" "
+ "configuration mismatch for device %s.",
+ cdev->dev->name);
+ goto error;
+ }
if (cdev->config.dbnc != config->dbnc) {
DRV_LOG(ERR, "\"" MLX5_SQ_DB_NC "\" "
"configuration mismatch for device %s.",
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 6cb40f54dd..f1b59d6f07 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -183,6 +183,7 @@ struct mlx5_dev_info {
uint32_t port_num;
uint32_t ibindex;
char ibname[MLX5_FS_NAME_MAX];
+ uint8_t probe_opt;
struct mlx5_port_nl_info *port_info;
};
@@ -525,6 +526,7 @@ struct mlx5_common_dev_config {
int pd_handle; /* Protection Domain handle for importation. */
unsigned int devx:1; /* Whether devx interface is available or not. */
unsigned int sys_mem_en:1; /* The default memory allocator. */
+ unsigned int probe_opt:1; /* Optimize probing . */
unsigned int mr_mempool_reg_en:1;
/* Allow/prevent implicit mempool memory registration. */
unsigned int mr_ext_memseg_en:1;
diff --git a/drivers/net/mlx5/linux/mlx5_ethdev_os.c b/drivers/net/mlx5/linux/mlx5_ethdev_os.c
index 08ac6dd939..88d3c57c6e 100644
--- a/drivers/net/mlx5/linux/mlx5_ethdev_os.c
+++ b/drivers/net/mlx5/linux/mlx5_ethdev_os.c
@@ -691,6 +691,8 @@ mlx5_handle_port_info_update(struct mlx5_dev_info *dev_info, uint32_t if_index,
if (dev_info->port_num <= 1 || dev_info->port_info == NULL)
return;
+ DRV_LOG(DEBUG, "IB device %s ifindex %u received netlink event %u",
+ dev_info->ibname, if_index, msg_type);
for (i = 1; i <= dev_info->port_num; i++) {
if (!dev_info->port_info[i].valid)
continue;
@@ -734,7 +736,8 @@ mlx5_dev_interrupt_nl_cb(struct nlmsghdr *hdr, void *cb_arg)
if (mlx5_nl_parse_link_status_update(hdr, &if_index) < 0)
return;
- mlx5_handle_port_info_update(&sh->cdev->dev_info, if_index, hdr->nlmsg_type);
+ if (sh->cdev->config.probe_opt && sh->cdev->dev_info.port_num > 1)
+ mlx5_handle_port_info_update(&sh->cdev->dev_info, if_index, hdr->nlmsg_type);
for (i = 0; i < sh->max_port; i++) {
struct mlx5_dev_shared_port *port = &sh->port[i];
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index dcf1ff917b..a408790d1e 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -2335,7 +2335,7 @@ mlx5_os_pci_probe_pf(struct mlx5_common_device *cdev,
while (ret-- > 0) {
struct rte_pci_addr pci_addr;
- if (cdev->dev_info.port_num) {
+ if (cdev->config.probe_opt && cdev->dev_info.port_num) {
if (strcmp(ibv_list[ret]->name, cdev->dev_info.ibname)) {
DRV_LOG(INFO, "Unmatched caching device \"%s\" \"%s\"",
cdev->dev_info.ibname, ibv_list[ret]->name);
--
2.34.1
next prev parent reply other threads:[~2024-10-16 8:39 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-10-16 8:38 [PATCH V1 0/7] port probe time optimization Minggang Li(Gavin)
2024-10-16 8:38 ` [PATCH V1 1/7] mailmap: update user name Minggang Li(Gavin)
2024-10-16 8:38 ` [PATCH V1 2/7] net/mlx5: optimize device probing Minggang Li(Gavin)
2024-10-16 8:38 ` Minggang Li(Gavin) [this message]
2024-10-16 8:38 ` [PATCH V1 4/7] common/mlx5: fix Netlink socket leak Minggang Li(Gavin)
2024-10-16 8:38 ` [PATCH V1 5/7] common/mlx5: add RDMA monitor event awareness Minggang Li(Gavin)
2024-10-16 8:38 ` [PATCH V1 6/7] mlx5: use RDMA Netlink to update port information Minggang Li(Gavin)
2024-10-16 8:38 ` [PATCH V1 7/7] mlx5: add backward compatibility for RDMA monitor Minggang Li(Gavin)
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20241016083818.662020-4-gavinl@nvidia.com \
--to=gavinl@nvidia.com \
--cc=bingz@nvidia.com \
--cc=dev@dpdk.org \
--cc=dsosnowski@nvidia.com \
--cc=matan@nvidia.com \
--cc=orika@nvidia.com \
--cc=rasland@nvidia.com \
--cc=rongweil@nvidia.com \
--cc=suanmingm@nvidia.com \
--cc=thomas@monjalon.net \
--cc=viacheslavo@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).