[dpdk-dev] [PATCH v3 3/5] vdpa/mlx5: workaround dirty bitmap MR creation

[Matan Azrad <matan@nvidia.com>]

Due to kernel driver/FW issues in direct MKEY creation using the DevX
API, this patch replaces the dirty bitmap MR creation to use wrapped
mkey instead.

Fixes: 9d39e57f21ac ("vdpa/mlx5: support live migration")
Cc: stable@dpdk.org

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/vdpa/mlx5/mlx5_vdpa.h     |  1 +
 drivers/vdpa/mlx5/mlx5_vdpa_lm.c  | 38 +++++++------------------------
 drivers/vdpa/mlx5/mlx5_vdpa_mem.c |  2 ++
 3 files changed, 11 insertions(+), 30 deletions(-)

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 62498f87fd..15212a2b30 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -147,6 +147,7 @@ struct mlx5_vdpa_priv {
 	struct mlx5_vdpa_steer steer;
 	struct mlx5dv_var *var;
 	void *virtq_db_addr;
+	struct mlx5_pmd_wrapped_mr lm_mr;
 	SLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list;
 	struct mlx5_vdpa_virtq virtqs[];
 };
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
index 3e8d9eb9a2..e65e4faa47 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
@@ -36,38 +36,22 @@ int
 mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
 			   uint64_t log_size)
 {
-	struct mlx5_devx_mkey_attr mkey_attr = {
-			.addr = (uintptr_t)log_base,
-			.size = log_size,
-			.pd = priv->cdev->pdn,
-			.pg_access = 1,
-	};
 	struct mlx5_devx_virtq_attr attr = {
 		.type = MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS,
 		.dirty_bitmap_addr = log_base,
 		.dirty_bitmap_size = log_size,
 	};
-	struct mlx5_vdpa_query_mr *mr = rte_malloc(__func__, sizeof(*mr), 0);
 	int i;
+	int ret = mlx5_os_wrapped_mkey_create(priv->cdev->ctx, priv->cdev->pd,
+					      priv->cdev->pdn,
+					      (void *)(uintptr_t)log_base,
+					      log_size, &priv->lm_mr);
 
-	if (!mr) {
-		DRV_LOG(ERR, "Failed to allocate mem for lm mr.");
+	if (!ret) {
+		DRV_LOG(ERR, "Failed to allocate wrapped MR for lm.");
 		return -1;
 	}
-	mr->umem = mlx5_glue->devx_umem_reg(priv->cdev->ctx,
-					    (void *)(uintptr_t)log_base,
-					    log_size, IBV_ACCESS_LOCAL_WRITE);
-	if (!mr->umem) {
-		DRV_LOG(ERR, "Failed to register umem for lm mr.");
-		goto err;
-	}
-	mkey_attr.umem_id = mr->umem->umem_id;
-	mr->mkey = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &mkey_attr);
-	if (!mr->mkey) {
-		DRV_LOG(ERR, "Failed to create Mkey for lm.");
-		goto err;
-	}
-	attr.dirty_bitmap_mkey = mr->mkey->id;
+	attr.dirty_bitmap_mkey = priv->lm_mr.lkey;
 	for (i = 0; i < priv->nr_virtqs; ++i) {
 		attr.queue_index = i;
 		if (!priv->virtqs[i].virtq) {
@@ -78,15 +62,9 @@ mlx5_vdpa_dirty_bitmap_set(struct mlx5_vdpa_priv *priv, uint64_t log_base,
 			goto err;
 		}
 	}
-	mr->is_indirect = 0;
-	SLIST_INSERT_HEAD(&priv->mr_list, mr, next);
 	return 0;
 err:
-	if (mr->mkey)
-		mlx5_devx_cmd_destroy(mr->mkey);
-	if (mr->umem)
-		mlx5_glue->devx_umem_dereg(mr->umem);
-	rte_free(mr);
+	mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
 	return -1;
 }
 
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index f551a094cd..d7707bbd91 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -31,6 +31,8 @@ mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
 		entry = next;
 	}
 	SLIST_INIT(&priv->mr_list);
+	if (priv->lm_mr.addr)
+		mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
 	if (priv->null_mr) {
 		claim_zero(mlx5_glue->dereg_mr(priv->null_mr));
 		priv->null_mr = NULL;
-- 
2.25.1