[RFC 15/15] vdpa/mlx5: prepare virtqueue resource creation

[Li Zhang <lizh@nvidia.com>]

Split the virtqs virt-queue resource between
the configuration threads.
Also need pre-created virt-queue resource
after virtq destruction.
This accelerates the LM process and reduces its time by 30%.

Signed-off-by: Li Zhang <lizh@nvidia.com>
---
 drivers/vdpa/mlx5/mlx5_vdpa.c         | 69 +++++++++++++++++++++++----
 drivers/vdpa/mlx5/mlx5_vdpa.h         |  7 ++-
 drivers/vdpa/mlx5/mlx5_vdpa_cthread.c | 14 +++++-
 drivers/vdpa/mlx5/mlx5_vdpa_virtq.c   | 35 ++++++++++----
 4 files changed, 104 insertions(+), 21 deletions(-)

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index eaca571e3e..15ce30bc49 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -275,13 +275,17 @@ mlx5_vdpa_wait_dev_close_tasks_done(struct mlx5_vdpa_priv *priv)
 }
 
 static int
-mlx5_vdpa_dev_close(int vid)
+_internal_mlx5_vdpa_dev_close(int vid, bool release_resource)
 {
 	struct rte_vdpa_device *vdev = rte_vhost_get_vdpa_device(vid);
-	struct mlx5_vdpa_priv *priv =
-		mlx5_vdpa_find_priv_resource_by_vdev(vdev);
+	struct mlx5_vdpa_priv *priv;
 	int ret = 0;
 
+	if (!vdev) {
+		DRV_LOG(ERR, "Invalid vDPA device.");
+		return -1;
+	}
+	priv = mlx5_vdpa_find_priv_resource_by_vdev(vdev);
 	if (priv == NULL) {
 		DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
 		return -1;
@@ -291,7 +295,7 @@ mlx5_vdpa_dev_close(int vid)
 		ret |= mlx5_vdpa_lm_log(priv);
 		priv->state = MLX5_VDPA_STATE_IN_PROGRESS;
 	}
-	if (priv->use_c_thread) {
+	if (priv->use_c_thread && !release_resource) {
 		if (priv->last_c_thrd_idx >=
 			(conf_thread_mng.max_thrds - 1))
 			priv->last_c_thrd_idx = 0;
@@ -315,7 +319,7 @@ mlx5_vdpa_dev_close(int vid)
 	pthread_mutex_lock(&priv->steer_update_lock);
 	mlx5_vdpa_steer_unset(priv);
 	pthread_mutex_unlock(&priv->steer_update_lock);
-	mlx5_vdpa_virtqs_release(priv);
+	mlx5_vdpa_virtqs_release(priv, release_resource);
 	mlx5_vdpa_drain_cq(priv);
 	if (priv->lm_mr.addr)
 		mlx5_os_wrapped_mkey_destroy(&priv->lm_mr);
@@ -329,6 +333,12 @@ mlx5_vdpa_dev_close(int vid)
 	return ret;
 }
 
+static int
+mlx5_vdpa_dev_close(int vid)
+{
+	return _internal_mlx5_vdpa_dev_close(vid, false);
+}
+
 static int
 mlx5_vdpa_dev_config(int vid)
 {
@@ -624,8 +634,9 @@ mlx5_vdpa_config_get(struct mlx5_kvargs_ctrl *mkvlist,
 static int
 mlx5_vdpa_virtq_resource_prepare(struct mlx5_vdpa_priv *priv)
 {
+	uint32_t remaining_cnt = 0, err_cnt = 0, task_num = 0;
 	uint32_t max_queues = priv->queues * 2;
-	uint32_t index;
+	uint32_t index, thrd_idx, data[1];
 	struct mlx5_vdpa_virtq *virtq;
 
 	for (index = 0; index < priv->caps.max_num_virtio_queues * 2;
@@ -635,10 +646,48 @@ mlx5_vdpa_virtq_resource_prepare(struct mlx5_vdpa_priv *priv)
 	}
 	if (!priv->queues || !priv->queue_size)
 		return 0;
-	for (index = 0; index < max_queues; ++index)
-		if (mlx5_vdpa_virtq_single_resource_prepare(priv,
-			index))
+	if (priv->use_c_thread) {
+		uint32_t main_task_idx[max_queues];
+
+		for (index = 0; index < max_queues; ++index) {
+			virtq = &priv->virtqs[index];
+			thrd_idx = index % (conf_thread_mng.max_thrds + 1);
+			if (!thrd_idx) {
+				main_task_idx[task_num] = index;
+				task_num++;
+				continue;
+			}
+			thrd_idx = priv->last_c_thrd_idx + 1;
+			if (thrd_idx >= conf_thread_mng.max_thrds)
+				thrd_idx = 0;
+			priv->last_c_thrd_idx = thrd_idx;
+			data[0] = index;
+			if (mlx5_vdpa_task_add(priv, thrd_idx,
+				MLX5_VDPA_TASK_PREPARE_VIRTQ,
+				&remaining_cnt, &err_cnt,
+				(void **)&data, 1)) {
+				DRV_LOG(ERR, "Fail to add "
+				"task prepare virtq (%d).", index);
+				main_task_idx[task_num] = index;
+				task_num++;
+			}
+		}
+		for (index = 0; index < task_num; ++index)
+			if (mlx5_vdpa_virtq_single_resource_prepare(priv,
+				main_task_idx[index]))
+				goto error;
+		if (mlx5_vdpa_c_thread_wait_bulk_tasks_done(&remaining_cnt,
+			&err_cnt, 2000)) {
+			DRV_LOG(ERR,
+			"Failed to wait virt-queue prepare tasks ready.");
 			goto error;
+		}
+	} else {
+		for (index = 0; index < max_queues; ++index)
+			if (mlx5_vdpa_virtq_single_resource_prepare(priv,
+				index))
+				goto error;
+	}
 	if (mlx5_vdpa_is_modify_virtq_supported(priv))
 		if (mlx5_vdpa_steer_update(priv, true))
 			goto error;
@@ -855,7 +904,7 @@ static void
 mlx5_vdpa_dev_release(struct mlx5_vdpa_priv *priv)
 {
 	if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
-		mlx5_vdpa_dev_close(priv->vid);
+		_internal_mlx5_vdpa_dev_close(priv->vid, true);
 	if (priv->use_c_thread)
 		mlx5_vdpa_wait_dev_close_tasks_done(priv);
 	mlx5_vdpa_release_dev_resources(priv);
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 00700261ec..477f2fdde0 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -85,6 +85,7 @@ enum mlx5_vdpa_task_type {
 	MLX5_VDPA_TASK_SETUP_VIRTQ,
 	MLX5_VDPA_TASK_STOP_VIRTQ,
 	MLX5_VDPA_TASK_DEV_CLOSE_NOWAIT,
+	MLX5_VDPA_TASK_PREPARE_VIRTQ,
 };
 
 /* Generic task information and size must be multiple of 4B. */
@@ -355,8 +356,12 @@ void mlx5_vdpa_err_event_unset(struct mlx5_vdpa_priv *priv);
  *
  * @param[in] priv
  *   The vdpa driver private structure.
+ * @param[in] release_resource
+ *   The vdpa driver realease resource without prepare resource.
  */
-void mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv);
+void
+mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv,
+		bool release_resource);
 
 /**
  * Cleanup cached resources of all virtqs.
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_cthread.c b/drivers/vdpa/mlx5/mlx5_vdpa_cthread.c
index 07efa0cb16..97109206d2 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_cthread.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_cthread.c
@@ -196,7 +196,7 @@ mlx5_vdpa_c_thread_handle(void *arg)
 			pthread_mutex_lock(&priv->steer_update_lock);
 			mlx5_vdpa_steer_unset(priv);
 			pthread_mutex_unlock(&priv->steer_update_lock);
-			mlx5_vdpa_virtqs_release(priv);
+			mlx5_vdpa_virtqs_release(priv, false);
 			mlx5_vdpa_drain_cq(priv);
 			if (priv->lm_mr.addr)
 				mlx5_os_wrapped_mkey_destroy(
@@ -208,6 +208,18 @@ mlx5_vdpa_c_thread_handle(void *arg)
 				&priv->dev_close_progress, 0,
 				__ATOMIC_RELAXED);
 			break;
+		case MLX5_VDPA_TASK_PREPARE_VIRTQ:
+			ret = mlx5_vdpa_virtq_single_resource_prepare(
+					priv, task.idx);
+			if (ret) {
+				DRV_LOG(ERR,
+				"Failed to prepare virtq %d.",
+				task.idx);
+				__atomic_fetch_add(
+				task.err_cnt, 1,
+				__ATOMIC_RELAXED);
+			}
+			break;
 		default:
 			DRV_LOG(ERR, "Invalid vdpa task type %d.",
 			task.type);
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
index 4a74738d9c..de6eab9bc6 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
@@ -113,6 +113,16 @@ mlx5_vdpa_virtq_unreg_intr_handle_all(struct mlx5_vdpa_priv *priv)
 	}
 }
 
+static void
+mlx5_vdpa_vq_destroy(struct mlx5_vdpa_virtq *virtq)
+{
+	/* Clean pre-created resource in dev removal only */
+	claim_zero(mlx5_devx_cmd_destroy(virtq->virtq));
+	virtq->index = 0;
+	virtq->virtq = NULL;
+	virtq->configured = 0;
+}
+
 /* Release cached VQ resources. */
 void
 mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv)
@@ -125,6 +135,8 @@ mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv)
 		if (virtq->index != i)
 			continue;
 		pthread_mutex_lock(&virtq->virtq_lock);
+		if (virtq->virtq)
+			mlx5_vdpa_vq_destroy(virtq);
 		for (j = 0; j < RTE_DIM(virtq->umems); ++j) {
 			if (virtq->umems[j].obj) {
 				claim_zero(mlx5_glue->devx_umem_dereg
@@ -154,29 +166,34 @@ mlx5_vdpa_virtq_unset(struct mlx5_vdpa_virtq *virtq)
 		if (ret)
 			DRV_LOG(WARNING, "Failed to stop virtq %d.",
 				virtq->index);
-		claim_zero(mlx5_devx_cmd_destroy(virtq->virtq));
-		virtq->index = 0;
-		virtq->virtq = NULL;
-		virtq->configured = 0;
+		mlx5_vdpa_vq_destroy(virtq);
 	}
 	virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_DISABLED;
 }
 
 void
-mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv)
+mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv,
+	bool release_resource)
 {
 	struct mlx5_vdpa_virtq *virtq;
-	int i;
+	uint32_t i, max_virtq;
 
-	for (i = 0; i < priv->nr_virtqs; i++) {
+	max_virtq = (release_resource &&
+		(priv->queues * 2) > priv->nr_virtqs) ?
+		(priv->queues * 2) : priv->nr_virtqs;
+	for (i = 0; i < max_virtq; i++) {
 		virtq = &priv->virtqs[i];
 		pthread_mutex_lock(&virtq->virtq_lock);
 		mlx5_vdpa_virtq_unset(virtq);
-		if (i < (priv->queues * 2))
+		if (!release_resource && i < (priv->queues * 2))
 			mlx5_vdpa_virtq_single_resource_prepare(
 					priv, i);
 		pthread_mutex_unlock(&virtq->virtq_lock);
 	}
+	if (!release_resource && priv->queues &&
+		mlx5_vdpa_is_modify_virtq_supported(priv))
+		if (mlx5_vdpa_steer_update(priv, true))
+			mlx5_vdpa_steer_unset(priv);
 	priv->features = 0;
 	priv->nr_virtqs = 0;
 }
@@ -733,7 +750,7 @@ mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv)
 	}
 	return 0;
 error:
-	mlx5_vdpa_virtqs_release(priv);
+	mlx5_vdpa_virtqs_release(priv, true);
 	return -1;
 }
 
-- 
2.27.0