From: Michael Baum <michaelba@nvidia.com>
To: <dev@dpdk.org>
Cc: Matan Azrad <matan@nvidia.com>,
Raslan Darawsheh <rasland@nvidia.com>,
Dariusz Sosnowski <dsosnowski@nvidia.com>,
Viacheslav Ovsiienko <viacheslavo@nvidia.com>,
Ori Kam <orika@nvidia.com>, Suanming Mou <suanmingm@nvidia.com>
Subject: [PATCH v2 12/23] net/mlx5: add physical device handle
Date: Thu, 25 Jan 2024 15:30:32 +0200 [thread overview]
Message-ID: <20240125133043.575860-13-michaelba@nvidia.com> (raw)
In-Reply-To: <20240125133043.575860-1-michaelba@nvidia.com>
Add structure describing physical device, and manage physical device
global list.
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
drivers/net/mlx5/mlx5.c | 77 ++++++++++++++++++++++++++++++++++++-----
drivers/net/mlx5/mlx5.h | 13 +++++++
2 files changed, 82 insertions(+), 8 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 3a182de248..f9fc652136 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -190,9 +190,10 @@ struct mlx5_shared_data *mlx5_shared_data;
/** Driver-specific log messages type. */
int mlx5_logtype;
-static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
- LIST_HEAD_INITIALIZER();
+static LIST_HEAD(mlx5_dev_ctx_list, mlx5_dev_ctx_shared) dev_ctx_list = LIST_HEAD_INITIALIZER();
+static LIST_HEAD(mlx5_phdev_list, mlx5_physical_device) phdev_list = LIST_HEAD_INITIALIZER();
static pthread_mutex_t mlx5_dev_ctx_list_mutex;
+
static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
[MLX5_IPOOL_DECAP_ENCAP] = {
@@ -1692,6 +1693,60 @@ mlx5_init_shared_dev_registers(struct mlx5_dev_ctx_shared *sh)
mlx5_init_hws_flow_tags_registers(sh);
}
+static struct mlx5_physical_device *
+mlx5_get_physical_device(struct mlx5_common_device *cdev)
+{
+ struct mlx5_physical_device *phdev;
+ struct mlx5_hca_attr *attr = &cdev->config.hca_attr;
+
+ /* Search for physical device by system_image_guid. */
+ LIST_FOREACH(phdev, &phdev_list, next) {
+ if (phdev->guid == attr->system_image_guid) {
+ phdev->refcnt++;
+ return phdev;
+ }
+ }
+ phdev = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
+ sizeof(struct mlx5_physical_device),
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+ if (!phdev) {
+ DRV_LOG(ERR, "Physical device allocation failure.");
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+ phdev->guid = attr->system_image_guid;
+ phdev->refcnt = 1;
+ LIST_INSERT_HEAD(&phdev_list, phdev, next);
+ DRV_LOG(DEBUG, "Physical device is created, guid=%" PRIu64 ".",
+ phdev->guid);
+ return phdev;
+}
+
+static void
+mlx5_physical_device_destroy(struct mlx5_physical_device *phdev)
+{
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+ /* Check the object presence in the list. */
+ struct mlx5_physical_device *lphdev;
+
+ LIST_FOREACH(lphdev, &phdev_list, next)
+ if (lphdev == phdev)
+ break;
+ MLX5_ASSERT(lphdev);
+ if (lphdev != phdev) {
+ DRV_LOG(ERR, "Freeing non-existing physical device");
+ return;
+ }
+#endif
+ MLX5_ASSERT(phdev);
+ MLX5_ASSERT(phdev->refcnt);
+ if (--phdev->refcnt)
+ return;
+ /* Remove physical device from the global device list. */
+ LIST_REMOVE(phdev, next);
+ mlx5_free(phdev);
+}
+
/**
* Allocate shared device context. If there is multiport device the
* master and representors will share this context, if there is single
@@ -1725,7 +1780,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
pthread_mutex_lock(&mlx5_dev_ctx_list_mutex);
/* Search for IB context by device name. */
- LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) {
+ LIST_FOREACH(sh, &dev_ctx_list, next) {
if (!strcmp(sh->ibdev_name, spawn->phys_dev_name)) {
sh->refcnt++;
goto exit;
@@ -1765,6 +1820,9 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
sizeof(sh->ibdev_name) - 1);
strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->cdev->ctx),
sizeof(sh->ibdev_path) - 1);
+ sh->phdev = mlx5_get_physical_device(sh->cdev);
+ if (!sh->phdev)
+ goto error;
/*
* Setting port_id to max unallowed value means there is no interrupt
* subhandler installed for the given port index i.
@@ -1798,7 +1856,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
#endif
}
mlx5_os_dev_shared_handler_install(sh);
- if (LIST_EMPTY(&mlx5_dev_ctx_list)) {
+ if (LIST_EMPTY(&dev_ctx_list)) {
err = mlx5_flow_os_init_workspace_once();
if (err)
goto error;
@@ -1811,7 +1869,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
mlx5_flow_aging_init(sh);
mlx5_flow_ipool_create(sh);
/* Add context to the global device list. */
- LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next);
+ LIST_INSERT_HEAD(&dev_ctx_list, sh, next);
rte_spinlock_init(&sh->geneve_tlv_opt_sl);
mlx5_init_shared_dev_registers(sh);
/* Init counter pool list header and lock. */
@@ -1833,6 +1891,8 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
} while (++i <= (uint32_t)sh->bond.n_port);
if (sh->td)
claim_zero(mlx5_devx_cmd_destroy(sh->td));
+ if (sh->phdev)
+ mlx5_physical_device_destroy(sh->phdev);
mlx5_free(sh);
rte_errno = err;
return NULL;
@@ -1919,7 +1979,7 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
/* Check the object presence in the list. */
struct mlx5_dev_ctx_shared *lctx;
- LIST_FOREACH(lctx, &mlx5_dev_ctx_list, next)
+ LIST_FOREACH(lctx, &dev_ctx_list, next)
if (lctx == sh)
break;
MLX5_ASSERT(lctx);
@@ -1945,7 +2005,7 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
/* Remove context from the global device list. */
LIST_REMOVE(sh, next);
/* Release resources on the last device removal. */
- if (LIST_EMPTY(&mlx5_dev_ctx_list)) {
+ if (LIST_EMPTY(&dev_ctx_list)) {
mlx5_os_net_cleanup();
mlx5_flow_os_release_workspace();
}
@@ -1985,6 +2045,7 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
MLX5_ASSERT(sh->geneve_tlv_option_resource == NULL);
pthread_mutex_destroy(&sh->txpp.mutex);
mlx5_lwm_unset(sh);
+ mlx5_physical_device_destroy(sh->phdev);
mlx5_free(sh);
return;
exit:
@@ -2929,7 +2990,7 @@ mlx5_probe_again_args_validate(struct mlx5_common_device *cdev,
return 0;
pthread_mutex_lock(&mlx5_dev_ctx_list_mutex);
/* Search for IB context by common device pointer. */
- LIST_FOREACH(sh, &mlx5_dev_ctx_list, next)
+ LIST_FOREACH(sh, &dev_ctx_list, next)
if (sh->cdev == cdev)
break;
pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 779805bcd8..8bf7f86416 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1419,6 +1419,18 @@ struct mlx5_dev_registers {
#define HAVE_MLX5_DR_CREATE_ACTION_ASO_EXT
#endif
+/**
+ * Physical device structure.
+ * This device is created once per NIC to manage recourses shared by all ports
+ * under same physical device.
+ */
+struct mlx5_physical_device {
+ LIST_ENTRY(mlx5_physical_device) next;
+ struct mlx5_dev_ctx_shared *sh; /* Created on sherd context. */
+ uint64_t guid; /* System image guid, the uniq ID of physical device. */
+ uint32_t refcnt;
+};
+
/*
* Shared Infiniband device context for Master/Representors
* which belong to same IB device with multiple IB ports.
@@ -1450,6 +1462,7 @@ struct mlx5_dev_ctx_shared {
uint32_t max_port; /* Maximal IB device port index. */
struct mlx5_bond_info bond; /* Bonding information. */
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
+ struct mlx5_physical_device *phdev; /* Backend physical device. */
uint32_t tdn; /* Transport Domain number. */
char ibdev_name[MLX5_FS_NAME_MAX]; /* SYSFS dev name. */
char ibdev_path[MLX5_FS_PATH_MAX]; /* SYSFS dev path for secondary */
--
2.25.1
next prev parent reply other threads:[~2024-01-25 13:32 UTC|newest]
Thread overview: 50+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-12-03 11:25 [PATCH v1 00/23] net/mlx5: support Geneve and options for HWS Michael Baum
2023-12-03 11:25 ` [PATCH v1 01/23] common/mlx5: fix duplicate read of general capabilities Michael Baum
2023-12-03 11:25 ` [PATCH v1 02/23] common/mlx5: fix query sample info capability Michael Baum
2023-12-03 11:25 ` [PATCH v1 03/23] net/mlx5/hws: fix tunnel protocol checks Michael Baum
2023-12-03 11:25 ` [PATCH v1 04/23] net/mlx5: remove GENEVE options length limitation Michael Baum
2023-12-03 11:25 ` [PATCH v1 05/23] net/mlx5: fix GENEVE option item translation Michael Baum
2023-12-03 11:25 ` [PATCH v1 06/23] common/mlx5: add system image GUID attribute Michael Baum
2023-12-03 11:25 ` [PATCH v1 07/23] common/mlx5: add GENEVE TLV option attribute structure Michael Baum
2023-12-03 11:25 ` [PATCH v1 08/23] common/mlx5: add PRM attribute for TLV sample Michael Baum
2023-12-03 11:25 ` [PATCH v1 09/23] common/mlx5: add sample info query syndrome into error log Michael Baum
2023-12-03 11:25 ` [PATCH v1 10/23] common/mlx5: query GENEVE option sample ID from HCA attr Michael Baum
2023-12-03 11:25 ` [PATCH v1 11/23] common/mlx5: add function to query GENEVE TLV option Michael Baum
2023-12-03 11:25 ` [PATCH v1 12/23] net/mlx5: add physical device handle Michael Baum
2023-12-03 11:25 ` [PATCH v1 13/23] net/mlx5: add GENEVE TLV options parser API Michael Baum
2023-12-03 11:25 ` [PATCH v1 14/23] net/mlx5: add API to expose GENEVE option FW information Michael Baum
2023-12-03 11:25 ` [PATCH v1 15/23] net/mlx5: add testpmd support for GENEVE TLV parser Michael Baum
2023-12-03 11:25 ` [PATCH v1 16/23] net/mlx5/hws: increase hl size for future compatibility Michael Baum
2023-12-03 11:25 ` [PATCH v1 17/23] net/mlx5/hws: support GENEVE matching Michael Baum
2023-12-03 11:25 ` [PATCH v1 18/23] net/mlx5/hws: support GENEVE options header Michael Baum
2023-12-03 11:25 ` [PATCH v1 19/23] net/mlx5: add support for GENEVE and option item in HWS Michael Baum
2023-12-03 11:25 ` [PATCH v1 20/23] net/mlx5: add GENEVE option support for profile 0 Michael Baum
2023-12-03 11:25 ` [PATCH v1 21/23] net/mlx5: add GENEVE option support for group 0 Michael Baum
2023-12-03 11:25 ` [PATCH v1 22/23] net/mlx5: add support for GENEVE VNI modify field Michael Baum
2023-12-03 11:25 ` [PATCH v1 23/23] net/mlx5: add support for modify GENEVE option header Michael Baum
2024-01-25 9:42 ` [PATCH v1 00/23] net/mlx5: support Geneve and options for HWS Suanming Mou
2024-01-25 13:30 ` [PATCH v2 " Michael Baum
2024-01-25 13:30 ` [PATCH v2 01/23] common/mlx5: fix duplicate read of general capabilities Michael Baum
2024-01-25 13:30 ` [PATCH v2 02/23] common/mlx5: fix query sample info capability Michael Baum
2024-01-25 13:30 ` [PATCH v2 03/23] net/mlx5/hws: fix tunnel protocol checks Michael Baum
2024-01-25 13:30 ` [PATCH v2 04/23] net/mlx5: remove GENEVE options length limitation Michael Baum
2024-01-25 13:30 ` [PATCH v2 05/23] net/mlx5: fix GENEVE option item translation Michael Baum
2024-01-25 13:30 ` [PATCH v2 06/23] common/mlx5: add system image GUID attribute Michael Baum
2024-01-25 13:30 ` [PATCH v2 07/23] common/mlx5: add GENEVE TLV option attribute structure Michael Baum
2024-01-25 13:30 ` [PATCH v2 08/23] common/mlx5: add PRM attribute for TLV sample Michael Baum
2024-01-25 13:30 ` [PATCH v2 09/23] common/mlx5: add sample info query syndrome into error log Michael Baum
2024-01-25 13:30 ` [PATCH v2 10/23] common/mlx5: query GENEVE option sample ID from HCA attr Michael Baum
2024-01-25 13:30 ` [PATCH v2 11/23] common/mlx5: add function to query GENEVE TLV option Michael Baum
2024-01-25 13:30 ` Michael Baum [this message]
2024-01-25 13:30 ` [PATCH v2 13/23] net/mlx5: add GENEVE TLV options parser API Michael Baum
2024-01-25 13:30 ` [PATCH v2 14/23] net/mlx5: add API to expose GENEVE option FW information Michael Baum
2024-01-25 13:30 ` [PATCH v2 15/23] net/mlx5: add testpmd support for GENEVE TLV parser Michael Baum
2024-01-25 13:30 ` [PATCH v2 16/23] net/mlx5/hws: increase hl size for future compatibility Michael Baum
2024-01-25 13:30 ` [PATCH v2 17/23] net/mlx5/hws: support GENEVE matching Michael Baum
2024-01-25 13:30 ` [PATCH v2 18/23] net/mlx5/hws: support GENEVE options header Michael Baum
2024-01-25 13:30 ` [PATCH v2 19/23] net/mlx5: add support for GENEVE and option item in HWS Michael Baum
2024-01-25 13:30 ` [PATCH v2 20/23] net/mlx5: add GENEVE option support for profile 0 Michael Baum
2024-01-25 13:30 ` [PATCH v2 21/23] net/mlx5: add GENEVE option support for group 0 Michael Baum
2024-01-25 13:30 ` [PATCH v2 22/23] net/mlx5: add support for GENEVE VNI modify field Michael Baum
2024-01-25 13:30 ` [PATCH v2 23/23] net/mlx5: add support for modify GENEVE option header Michael Baum
2024-01-29 12:21 ` [PATCH v2 00/23] net/mlx5: support Geneve and options for HWS Raslan Darawsheh
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240125133043.575860-13-michaelba@nvidia.com \
--to=michaelba@nvidia.com \
--cc=dev@dpdk.org \
--cc=dsosnowski@nvidia.com \
--cc=matan@nvidia.com \
--cc=orika@nvidia.com \
--cc=rasland@nvidia.com \
--cc=suanmingm@nvidia.com \
--cc=viacheslavo@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).