From: Ophir Munk <ophirmu@mellanox.com>
To: dev@dpdk.org, Matan Azrad <matan@mellanox.com>,
Raslan Darawsheh <rasland@mellanox.com>
Cc: Ophir Munk <ophirmu@mellanox.com>
Subject: [dpdk-dev] [PATCH v1 2/4] common/mlx5: export MR verbs operations
Date: Tue, 16 Jun 2020 09:44:44 +0000 [thread overview]
Message-ID: <20200616094446.7152-3-ophirmu@mellanox.com> (raw)
In-Reply-To: <20200616094446.7152-1-ophirmu@mellanox.com>
The glue verbs operations reg_mr and dereg_mr are wrapped and exported
in functions mlx5_common_verbs_reg_mr and mlx5_common_verbs_dereg_mr
respectively. The exported functions are added to a new file
linux/mlx5_common_verbs.c.
Signed-off-by: Ophir Munk <ophirmu@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
drivers/common/mlx5/Makefile | 1 +
drivers/common/mlx5/linux/meson.build | 1 +
drivers/common/mlx5/linux/mlx5_common_verbs.c | 87 +++++++++++++++++++++++++
drivers/common/mlx5/mlx5_common_mr.c | 27 ++------
drivers/common/mlx5/mlx5_common_mr.h | 8 ++-
drivers/common/mlx5/rte_common_mlx5_version.map | 3 +
6 files changed, 104 insertions(+), 23 deletions(-)
create mode 100644 drivers/common/mlx5/linux/mlx5_common_verbs.c
diff --git a/drivers/common/mlx5/Makefile b/drivers/common/mlx5/Makefile
index 4948fef..622bde4 100644
--- a/drivers/common/mlx5/Makefile
+++ b/drivers/common/mlx5/Makefile
@@ -18,6 +18,7 @@ SRCS-y += mlx5_devx_cmds.c
SRCS-y += mlx5_common.c
SRCS-y += linux/mlx5_common_os.c
SRCS-y += linux/mlx5_nl.c
+SRCS-y += linux/mlx5_common_verbs.c
SRCS-y += mlx5_common_mp.c
SRCS-y += mlx5_common_mr.c
ifeq ($(CONFIG_RTE_IBVERBS_LINK_DLOPEN),y)
diff --git a/drivers/common/mlx5/linux/meson.build b/drivers/common/mlx5/linux/meson.build
index ce065d9..638bb2b 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -47,6 +47,7 @@ endif
sources += files('mlx5_nl.c')
sources += files('mlx5_common_os.c')
+sources += files('mlx5_common_verbs.c')
if not dlopen_ibverbs
sources += files('mlx5_glue.c')
endif
diff --git a/drivers/common/mlx5/linux/mlx5_common_verbs.c b/drivers/common/mlx5/linux/mlx5_common_verbs.c
new file mode 100644
index 0000000..e7e50ad
--- /dev/null
+++ b/drivers/common/mlx5/linux/mlx5_common_verbs.c
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include <stddef.h>
+#include <errno.h>
+#include <string.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <inttypes.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#include "mlx5_autoconf.h"
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#ifdef HAVE_INFINIBAND_VERBS_H
+#include <infiniband/verbs.h>
+#endif
+#ifdef HAVE_INFINIBAND_MLX5DV_H
+#include <infiniband/mlx5dv.h>
+#endif
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_mr.h>
+
+/**
+ * Register mr. Given protection doamin pointer, pointer to addr and length
+ * register the memory region.
+ *
+ * @param[in] pd
+ * Pointer to protection domain context.
+ * @param[in] addr
+ * Pointer to memory start address.
+ * @param[in] lentgh
+ * Length of the memory to register.
+ * @param[out] pmd_mr
+ * pmd_mr struct set with lkey, address, length and pointer to mr object
+ *
+ * @return
+ * 0 on successful registration, -1 otherwise
+ */
+int
+mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr)
+{
+ struct ibv_mr *ibv_mr;
+
+ memset(pmd_mr, 0, sizeof(*pmd_mr));
+ ibv_mr = mlx5_glue->reg_mr(pd, addr, length,
+ IBV_ACCESS_LOCAL_WRITE |
+ (haswell_broadwell_cpu ? 0 :
+ IBV_ACCESS_RELAXED_ORDERING));
+ if (!ibv_mr)
+ return -1;
+
+ *pmd_mr = (struct mlx5_pmd_mr){
+ .lkey = ibv_mr->lkey,
+ .addr = ibv_mr->addr,
+ .len = ibv_mr->length,
+ .obj = (void *)ibv_mr,
+ };
+ return 0;
+}
+
+/**
+ * Deregister mr. Given the mlx5 pmd MR - deregister the MR
+ *
+ * @param[in] pmd_mr
+ * pmd_mr struct set with lkey, address, length and pointer to mr object
+ *
+ */
+void
+mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
+{
+ if (pmd_mr && pmd_mr->obj != NULL) {
+ claim_zero(mlx5_glue->dereg_mr(pmd_mr->obj));
+ memset(pmd_mr, 0, sizeof(*pmd_mr));
+ }
+}
+
diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c
index c68be54..4ce7975 100644
--- a/drivers/common/mlx5/mlx5_common_mr.c
+++ b/drivers/common/mlx5/mlx5_common_mr.c
@@ -440,8 +440,7 @@ mr_free(struct mlx5_mr *mr)
if (mr == NULL)
return;
DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
- if (mr->pmd_mr.obj != NULL)
- claim_zero(mlx5_glue->dereg_mr(mr->pmd_mr.obj));
+ mlx5_common_verbs_dereg_mr(&mr->pmd_mr);
if (mr->ms_bmp != NULL)
rte_bitmap_free(mr->ms_bmp);
rte_free(mr);
@@ -598,7 +597,6 @@ mlx5_mr_create_primary(void *pd,
uint32_t ms_n;
uint32_t n;
size_t len;
- struct ibv_mr *ibv_mr;
DRV_LOG(DEBUG, "Creating a MR using address (%p)", (void *)addr);
/*
@@ -767,20 +765,13 @@ mlx5_mr_create_primary(void *pd,
* mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
* through mlx5_alloc_verbs_buf().
*/
- ibv_mr = mlx5_glue->reg_mr(pd, (void *)data.start, len,
- IBV_ACCESS_LOCAL_WRITE |
- (haswell_broadwell_cpu ? 0 :
- IBV_ACCESS_RELAXED_ORDERING));
- if (ibv_mr == NULL) {
+ mlx5_common_verbs_reg_mr(pd, (void *)data.start, len, &mr->pmd_mr);
+ if (mr->pmd_mr.obj == NULL) {
DEBUG("Fail to create an MR for address (%p)",
(void *)addr);
rte_errno = EINVAL;
goto err_mrlock;
}
- mr->pmd_mr.lkey = ibv_mr->lkey;
- mr->pmd_mr.addr = ibv_mr->addr;
- mr->pmd_mr.len = ibv_mr->length;
- mr->pmd_mr.obj = ibv_mr;
MLX5_ASSERT((uintptr_t)mr->pmd_mr.addr == data.start);
MLX5_ASSERT(mr->pmd_mr.len);
LIST_INSERT_HEAD(&share_cache->mr_list, mr, mr);
@@ -1039,7 +1030,6 @@ mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
struct mlx5_mr *
mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id)
{
- struct ibv_mr *ibv_mr;
struct mlx5_mr *mr = NULL;
mr = rte_zmalloc_socket(NULL,
@@ -1048,21 +1038,14 @@ mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id)
RTE_CACHE_LINE_SIZE, socket_id);
if (mr == NULL)
return NULL;
- ibv_mr = mlx5_glue->reg_mr(pd, (void *)addr, len,
- IBV_ACCESS_LOCAL_WRITE |
- (haswell_broadwell_cpu ? 0 :
- IBV_ACCESS_RELAXED_ORDERING));
- if (ibv_mr == NULL) {
+ mlx5_common_verbs_reg_mr(pd, (void *)addr, len, &mr->pmd_mr);
+ if (mr->pmd_mr.obj == NULL) {
DRV_LOG(WARNING,
"Fail to create MR for address (%p)",
(void *)addr);
rte_free(mr);
return NULL;
}
- mr->pmd_mr.lkey = ibv_mr->lkey;
- mr->pmd_mr.addr = ibv_mr->addr;
- mr->pmd_mr.len = ibv_mr->length;
- mr->pmd_mr.obj = ibv_mr;
mr->msl = NULL; /* Mark it is external memory. */
mr->ms_bmp = NULL;
mr->ms_n = 1;
diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h
index 6f2dcec..cdb8acf 100644
--- a/drivers/common/mlx5/mlx5_common_mr.h
+++ b/drivers/common/mlx5/mlx5_common_mr.h
@@ -163,5 +163,11 @@ mlx5_mr_create_primary(void *pd,
struct mlx5_mr_share_cache *share_cache,
struct mr_cache_entry *entry, uintptr_t addr,
unsigned int mr_ext_memseg_en);
-
+__rte_internal
+int
+mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
+ struct mlx5_pmd_mr *pmd_mr);
+__rte_internal
+void
+mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
#endif /* RTE_PMD_MLX5_COMMON_MR_H_ */
diff --git a/drivers/common/mlx5/rte_common_mlx5_version.map b/drivers/common/mlx5/rte_common_mlx5_version.map
index 350e771..68f1207 100644
--- a/drivers/common/mlx5/rte_common_mlx5_version.map
+++ b/drivers/common/mlx5/rte_common_mlx5_version.map
@@ -3,6 +3,9 @@ INTERNAL {
mlx5_class_get;
+ mlx5_common_verbs_reg_mr;
+ mlx5_common_verbs_dereg_mr;
+
mlx5_create_mr_ext;
mlx5_dev_to_pci_addr;
--
2.8.4
next prev parent reply other threads:[~2020-06-16 9:45 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-06-16 9:44 [dpdk-dev] [PATCH v1 0/4] mlx5 MR refactor Ophir Munk
2020-06-16 9:44 ` [dpdk-dev] [PATCH v1 1/4] common/mlx5: remove MR struct dependency on ibv Ophir Munk
2020-06-16 9:44 ` Ophir Munk [this message]
2020-06-16 9:44 ` [dpdk-dev] [PATCH v1 3/4] net/mlx5: add MR callbacks in per-device cache Ophir Munk
2020-06-16 9:44 ` [dpdk-dev] [PATCH v1 4/4] mlx5/linux: add MR callbacks to verbs operations struct Ophir Munk
2020-06-17 10:13 ` [dpdk-dev] [PATCH v1 0/4] mlx5 MR refactor Raslan Darawsheh
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200616094446.7152-3-ophirmu@mellanox.com \
--to=ophirmu@mellanox.com \
--cc=dev@dpdk.org \
--cc=matan@mellanox.com \
--cc=rasland@mellanox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).