From: Matan Azrad <matan@nvidia.com>
To: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
Cc: <dev@dpdk.org>, Thomas Monjalon <thomas@monjalon.net>,
Michael Baum <michaelba@oss.nvidia.com>, <stable@dpdk.org>
Subject: [dpdk-dev] [PATCH v2 4/5] vdpa/mlx5: workaround guest MR registrations
Date: Mon, 8 Nov 2021 19:21:12 +0200 [thread overview]
Message-ID: <20211108172113.2241853-5-matan@nvidia.com> (raw)
In-Reply-To: <20211108172113.2241853-1-matan@nvidia.com>
From: Michael Baum <michaelba@oss.nvidia.com>
Due to kernel issue in direct MKEY creation using the DevX API, this
patch replaces the virtio MR creation to use Verbs API.
Fixes: cc07a42da250 ("vdpa/mlx5: prepare memory regions")
Cc: stable@dpdk.org
Signed-off-by: Michael Baum <michaelba@oss.nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
drivers/vdpa/mlx5/mlx5_vdpa.h | 8 +++---
drivers/vdpa/mlx5/mlx5_vdpa_mem.c | 41 +++++++++----------------------
2 files changed, 16 insertions(+), 33 deletions(-)
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 3a7cf088b8..f290fb4895 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -59,10 +59,10 @@ struct mlx5_vdpa_event_qp {
struct mlx5_vdpa_query_mr {
SLIST_ENTRY(mlx5_vdpa_query_mr) next;
- void *addr;
- uint64_t length;
- struct mlx5dv_devx_umem *umem;
- struct mlx5_devx_obj *mkey;
+ union {
+ struct ibv_mr *mr;
+ struct mlx5_devx_obj *mkey;
+ };
int is_indirect;
};
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index d7707bbd91..b1b9053bff 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -23,9 +23,10 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
entry = SLIST_FIRST(&priv->mr_list);
while (entry) {
next = SLIST_NEXT(entry, next);
- claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
- if (!entry->is_indirect)
- claim_zero(mlx5_glue->devx_umem_dereg(entry->umem));
+ if (entry->is_indirect)
+ claim_zero(mlx5_devx_cmd_destroy(entry->mkey));
+ else
+ claim_zero(mlx5_glue->dereg_mr(entry->mr));
SLIST_REMOVE(&priv->mr_list, entry, mlx5_vdpa_query_mr, next);
rte_free(entry);
entry = next;
@@ -202,7 +203,6 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
goto error;
}
DRV_LOG(DEBUG, "Dump fill Mkey = %u.", priv->null_mr->lkey);
- memset(&mkey_attr, 0, sizeof(mkey_attr));
for (i = 0; i < mem->nregions; i++) {
reg = &mem->regions[i];
entry = rte_zmalloc(__func__, sizeof(*entry), 0);
@@ -211,28 +211,15 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
DRV_LOG(ERR, "Failed to allocate mem entry memory.");
goto error;
}
- entry->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
- (void *)(uintptr_t)reg->host_user_addr,
- reg->size, IBV_ACCESS_LOCAL_WRITE);
- if (!entry->umem) {
- DRV_LOG(ERR, "Failed to register Umem by Devx.");
- ret = -errno;
- goto error;
- }
- mkey_attr.addr = (uintptr_t)(reg->guest_phys_addr);
- mkey_attr.size = reg->size;
- mkey_attr.umem_id = entry->umem->umem_id;
- mkey_attr.pd = priv->cdev->pdn;
- mkey_attr.pg_access = 1;
- entry->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx,
- &mkey_attr);
- if (!entry->mkey) {
+ entry->mr = mlx5_glue->reg_mr_iova(priv->cdev->pd,
+ (void *)(uintptr_t)(reg->host_user_addr),
+ reg->size, reg->guest_phys_addr,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!entry->mr) {
DRV_LOG(ERR, "Failed to create direct Mkey.");
ret = -rte_errno;
goto error;
}
- entry->addr = (void *)(uintptr_t)(reg->host_user_addr);
- entry->length = reg->size;
entry->is_indirect = 0;
if (i > 0) {
uint64_t sadd;
@@ -262,12 +249,13 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
for (k = 0; k < reg->size; k += klm_size) {
klm_array[klm_index].byte_count = k + klm_size >
reg->size ? reg->size - k : klm_size;
- klm_array[klm_index].mkey = entry->mkey->id;
+ klm_array[klm_index].mkey = entry->mr->lkey;
klm_array[klm_index].address = reg->guest_phys_addr + k;
klm_index++;
}
SLIST_INSERT_HEAD(&priv->mr_list, entry, next);
}
+ memset(&mkey_attr, 0, sizeof(mkey_attr));
mkey_attr.addr = (uintptr_t)(mem->regions[0].guest_phys_addr);
mkey_attr.size = mem_size;
mkey_attr.pd = priv->cdev->pdn;
@@ -295,13 +283,8 @@ mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv)
priv->gpa_mkey_index = entry->mkey->id;
return 0;
error:
- if (entry) {
- if (entry->mkey)
- mlx5_devx_cmd_destroy(entry->mkey);
- if (entry->umem)
- mlx5_glue->devx_umem_dereg(entry->umem);
+ if (entry)
rte_free(entry);
- }
mlx5_vdpa_mem_dereg(priv);
rte_errno = -ret;
return ret;
--
2.25.1
next prev parent reply other threads:[~2021-11-08 17:22 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-11-07 15:29 [dpdk-dev] [PATCH 0/5] mlx5: workaround MR issues in FW\kernel Matan Azrad
2021-11-07 15:29 ` [dpdk-dev] [PATCH 1/5] common/mlx5: glue MR registration with IOVA Matan Azrad
2021-11-07 15:29 ` [dpdk-dev] [PATCH 2/5] common/mlx5: add wrapped MR create API Matan Azrad
2021-11-07 15:29 ` [dpdk-dev] [PATCH 3/5] vdpa/mlx5: workaround dirty bitmap MR creation Matan Azrad
2021-11-07 15:29 ` [dpdk-dev] [PATCH 4/5] vdpa/mlx5: workaround guest MR registrations Matan Azrad
2021-11-07 15:29 ` [dpdk-dev] [PATCH 5/5] net/mlx5: workaround counter memory region creation Matan Azrad
2021-11-08 17:21 ` [dpdk-dev] [PATCH v2 0/5] mlx5: workaround MR issues Matan Azrad
2021-11-08 17:21 ` [dpdk-dev] [PATCH v2 1/5] common/mlx5: glue MR registration with IOVA Matan Azrad
2021-11-08 17:21 ` [dpdk-dev] [PATCH v2 2/5] common/mlx5: add wrapped MR create API Matan Azrad
2021-11-08 17:21 ` [dpdk-dev] [PATCH v2 3/5] vdpa/mlx5: workaround dirty bitmap MR creation Matan Azrad
2021-11-08 19:38 ` [dpdk-dev] [dpdk-stable] " Thomas Monjalon
2021-11-08 17:21 ` Matan Azrad [this message]
2021-11-08 17:21 ` [dpdk-dev] [PATCH v2 5/5] net/mlx5: workaround MR creation for flow counter Matan Azrad
2021-11-09 12:23 ` [dpdk-dev] [PATCH v3 0/5] mlx5: workaround MR issues Matan Azrad
2021-11-09 12:36 ` Matan Azrad
2021-11-09 12:36 ` [dpdk-dev] [PATCH v3 1/5] common/mlx5: glue MR registration with IOVA Matan Azrad
2021-11-09 12:36 ` [dpdk-dev] [PATCH v3 2/5] common/mlx5: add wrapped MR create API Matan Azrad
2021-11-09 12:36 ` [dpdk-dev] [PATCH v3 3/5] vdpa/mlx5: workaround dirty bitmap MR creation Matan Azrad
2021-11-09 12:36 ` [dpdk-dev] [PATCH v3 4/5] vdpa/mlx5: workaround guest MR registrations Matan Azrad
2021-11-09 12:36 ` [dpdk-dev] [PATCH v3 5/5] net/mlx5: workaround MR creation for flow counter Matan Azrad
2021-11-10 14:55 ` [dpdk-dev] [PATCH v3 0/5] mlx5: workaround MR issues Thomas Monjalon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211108172113.2241853-5-matan@nvidia.com \
--to=matan@nvidia.com \
--cc=dev@dpdk.org \
--cc=michaelba@oss.nvidia.com \
--cc=stable@dpdk.org \
--cc=thomas@monjalon.net \
--cc=viacheslavo@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).