From: Ophir Munk <ophirmu@mellanox.com>
To: dev@dpdk.org, Matan Azrad <matan@mellanox.com>,
Raslan Darawsheh <rasland@mellanox.com>
Cc: Ophir Munk <ophirmu@mellanox.com>
Subject: [dpdk-dev] [PATCH v1 1/4] common/mlx5: remove MR struct dependency on ibv
Date: Tue, 16 Jun 2020 09:44:43 +0000 [thread overview]
Message-ID: <20200616094446.7152-2-ophirmu@mellanox.com> (raw)
In-Reply-To: <20200616094446.7152-1-ophirmu@mellanox.com>
Replace 'struct ibv_mr *' (in 'struct mlx5_mr') with a new 'struct
mlx5_pmd_mr'. The new struct contains the required MR field: lkey,
addr, len and is independent of ibv.
Signed-off-by: Ophir Munk <ophirmu@mellanox.com>
Acked-by: Matan Azrad <matan@mellanox.com>
---
drivers/common/mlx5/mlx5_common_mr.c | 60 ++++++++++++++++++++----------------
drivers/common/mlx5/mlx5_common_mr.h | 11 +++++--
2 files changed, 43 insertions(+), 28 deletions(-)
diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c
index e894523..c68be54 100644
--- a/drivers/common/mlx5/mlx5_common_mr.c
+++ b/drivers/common/mlx5/mlx5_common_mr.c
@@ -266,18 +266,16 @@ mr_find_next_chunk(struct mlx5_mr *mr, struct mr_cache_entry *entry,
/* MR for external memory doesn't have memseg list. */
if (mr->msl == NULL) {
- struct ibv_mr *ibv_mr = mr->ibv_mr;
-
MLX5_ASSERT(mr->ms_bmp_n == 1);
MLX5_ASSERT(mr->ms_n == 1);
MLX5_ASSERT(base_idx == 0);
/*
* Can't search it from memseg list but get it directly from
- * verbs MR as there's only one chunk.
+ * pmd_mr as there's only one chunk.
*/
- entry->start = (uintptr_t)ibv_mr->addr;
- entry->end = (uintptr_t)ibv_mr->addr + mr->ibv_mr->length;
- entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+ entry->start = (uintptr_t)mr->pmd_mr.addr;
+ entry->end = (uintptr_t)mr->pmd_mr.addr + mr->pmd_mr.len;
+ entry->lkey = rte_cpu_to_be_32(mr->pmd_mr.lkey);
/* Returning 1 ends iteration. */
return 1;
}
@@ -302,7 +300,7 @@ mr_find_next_chunk(struct mlx5_mr *mr, struct mr_cache_entry *entry,
/* Found one chunk. */
entry->start = start;
entry->end = end;
- entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+ entry->lkey = rte_cpu_to_be_32(mr->pmd_mr.lkey);
}
return idx;
}
@@ -442,8 +440,8 @@ mr_free(struct mlx5_mr *mr)
if (mr == NULL)
return;
DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
- if (mr->ibv_mr != NULL)
- claim_zero(mlx5_glue->dereg_mr(mr->ibv_mr));
+ if (mr->pmd_mr.obj != NULL)
+ claim_zero(mlx5_glue->dereg_mr(mr->pmd_mr.obj));
if (mr->ms_bmp != NULL)
rte_bitmap_free(mr->ms_bmp);
rte_free(mr);
@@ -600,6 +598,7 @@ mlx5_mr_create_primary(void *pd,
uint32_t ms_n;
uint32_t n;
size_t len;
+ struct ibv_mr *ibv_mr;
DRV_LOG(DEBUG, "Creating a MR using address (%p)", (void *)addr);
/*
@@ -768,24 +767,28 @@ mlx5_mr_create_primary(void *pd,
* mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
* through mlx5_alloc_verbs_buf().
*/
- mr->ibv_mr = mlx5_glue->reg_mr(pd, (void *)data.start, len,
- IBV_ACCESS_LOCAL_WRITE |
- (haswell_broadwell_cpu ? 0 :
- IBV_ACCESS_RELAXED_ORDERING));
- if (mr->ibv_mr == NULL) {
- DEBUG("Fail to create a verbs MR for address (%p)",
+ ibv_mr = mlx5_glue->reg_mr(pd, (void *)data.start, len,
+ IBV_ACCESS_LOCAL_WRITE |
+ (haswell_broadwell_cpu ? 0 :
+ IBV_ACCESS_RELAXED_ORDERING));
+ if (ibv_mr == NULL) {
+ DEBUG("Fail to create an MR for address (%p)",
(void *)addr);
rte_errno = EINVAL;
goto err_mrlock;
}
- MLX5_ASSERT((uintptr_t)mr->ibv_mr->addr == data.start);
- MLX5_ASSERT(mr->ibv_mr->length == len);
+ mr->pmd_mr.lkey = ibv_mr->lkey;
+ mr->pmd_mr.addr = ibv_mr->addr;
+ mr->pmd_mr.len = ibv_mr->length;
+ mr->pmd_mr.obj = ibv_mr;
+ MLX5_ASSERT((uintptr_t)mr->pmd_mr.addr == data.start);
+ MLX5_ASSERT(mr->pmd_mr.len);
LIST_INSERT_HEAD(&share_cache->mr_list, mr, mr);
DEBUG("MR CREATED (%p) for %p:\n"
" [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
(void *)mr, (void *)addr, data.start, data.end,
- rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ rte_cpu_to_be_32(mr->pmd_mr.lkey),
mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
/* Insert to the global cache table. */
mlx5_mr_insert_cache(share_cache, mr);
@@ -1036,6 +1039,7 @@ mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
struct mlx5_mr *
mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id)
{
+ struct ibv_mr *ibv_mr;
struct mlx5_mr *mr = NULL;
mr = rte_zmalloc_socket(NULL,
@@ -1044,17 +1048,21 @@ mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id)
RTE_CACHE_LINE_SIZE, socket_id);
if (mr == NULL)
return NULL;
- mr->ibv_mr = mlx5_glue->reg_mr(pd, (void *)addr, len,
- IBV_ACCESS_LOCAL_WRITE |
- (haswell_broadwell_cpu ? 0 :
- IBV_ACCESS_RELAXED_ORDERING));
- if (mr->ibv_mr == NULL) {
+ ibv_mr = mlx5_glue->reg_mr(pd, (void *)addr, len,
+ IBV_ACCESS_LOCAL_WRITE |
+ (haswell_broadwell_cpu ? 0 :
+ IBV_ACCESS_RELAXED_ORDERING));
+ if (ibv_mr == NULL) {
DRV_LOG(WARNING,
- "Fail to create a verbs MR for address (%p)",
+ "Fail to create MR for address (%p)",
(void *)addr);
rte_free(mr);
return NULL;
}
+ mr->pmd_mr.lkey = ibv_mr->lkey;
+ mr->pmd_mr.addr = ibv_mr->addr;
+ mr->pmd_mr.len = ibv_mr->length;
+ mr->pmd_mr.obj = ibv_mr;
mr->msl = NULL; /* Mark it is external memory. */
mr->ms_bmp = NULL;
mr->ms_n = 1;
@@ -1064,7 +1072,7 @@ mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id)
" [0x%" PRIxPTR ", 0x%" PRIxPTR "),"
" lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
(void *)mr, (void *)addr,
- addr, addr + len, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ addr, addr + len, rte_cpu_to_be_32(mr->pmd_mr.lkey),
mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
return mr;
}
@@ -1089,7 +1097,7 @@ mlx5_mr_dump_cache(struct mlx5_mr_share_cache *share_cache __rte_unused)
unsigned int n;
DEBUG("MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
- mr_n++, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr_n++, rte_cpu_to_be_32(mr->pmd_mr.lkey),
mr->ms_n, mr->ms_bmp_n);
if (mr->ms_n == 0)
continue;
diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h
index 7add4da..6f2dcec 100644
--- a/drivers/common/mlx5/mlx5_common_mr.h
+++ b/drivers/common/mlx5/mlx5_common_mr.h
@@ -31,10 +31,17 @@
#define MLX5_MR_CACHE_N 8
#define MLX5_MR_BTREE_CACHE_N 256
+/* mlx5 PMD MR struct. */
+struct mlx5_pmd_mr {
+ uint32_t lkey;
+ void *addr;
+ size_t len;
+ void *obj; /* verbs mr object or devx umem object. */
+};
/* Memory Region object. */
struct mlx5_mr {
LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
- struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
+ struct mlx5_pmd_mr pmd_mr; /* PMD memory region. */
const struct rte_memseg_list *msl;
int ms_base_idx; /* Start index of msl->memseg_arr[]. */
int ms_n; /* Number of memsegs in use. */
@@ -46,7 +53,7 @@ struct mlx5_mr {
struct mr_cache_entry {
uintptr_t start; /* Start address of MR. */
uintptr_t end; /* End address of MR. */
- uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
+ uint32_t lkey; /* rte_cpu_to_be_32(lkey). */
} __rte_packed;
/* MR Cache table for Binary search. */
--
2.8.4
next prev parent reply other threads:[~2020-06-16 9:45 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-06-16 9:44 [dpdk-dev] [PATCH v1 0/4] mlx5 MR refactor Ophir Munk
2020-06-16 9:44 ` Ophir Munk [this message]
2020-06-16 9:44 ` [dpdk-dev] [PATCH v1 2/4] common/mlx5: export MR verbs operations Ophir Munk
2020-06-16 9:44 ` [dpdk-dev] [PATCH v1 3/4] net/mlx5: add MR callbacks in per-device cache Ophir Munk
2020-06-16 9:44 ` [dpdk-dev] [PATCH v1 4/4] mlx5/linux: add MR callbacks to verbs operations struct Ophir Munk
2020-06-17 10:13 ` [dpdk-dev] [PATCH v1 0/4] mlx5 MR refactor Raslan Darawsheh
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200616094446.7152-2-ophirmu@mellanox.com \
--to=ophirmu@mellanox.com \
--cc=dev@dpdk.org \
--cc=matan@mellanox.com \
--cc=rasland@mellanox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).