DPDK patches and discussions
 help / color / mirror / Atom feed
From: Matan Azrad <matan@mellanox.com>
To: Maxime Coquelin <maxime.coquelin@redhat.com>
Cc: dev@dpdk.org, Xiao Wang <xiao.w.wang@intel.com>
Subject: [dpdk-dev] [PATCH v3 6/6] vdpa/mlx5: support queue update
Date: Mon, 29 Jun 2020 14:08:21 +0000
Message-ID: <1593439701-136089-7-git-send-email-matan@mellanox.com> (raw)
In-Reply-To: <1593439701-136089-1-git-send-email-matan@mellanox.com>

Last changes in vDPA device management by vhost library may cause queue
ready state update after the device configuration.

So, there is chance that some queue configuration information will be
known only after the device was configured.

Add support to reconfigure a queue after the device configuration
according to the queue state update and the configuration changes.

Adjust the host notifier and the guest notification configuration to be
per queue and to be applied in the enablement process.

Signed-off-by: Matan Azrad <matan@mellanox.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/mlx5/mlx5_vdpa.c       | 26 ------------
 drivers/vdpa/mlx5/mlx5_vdpa.h       |  8 +++-
 drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 80 ++++++++++++++++++++++++++-----------
 3 files changed, 64 insertions(+), 50 deletions(-)

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index 97f87c5..2aa168d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -141,31 +141,6 @@
 }
 
 static int
-mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv)
-{
-	int ret;
-
-	if (priv->direct_notifier) {
-		ret = rte_vhost_host_notifier_ctrl(priv->vid,
-						   RTE_VHOST_QUEUE_ALL, false);
-		if (ret != 0) {
-			DRV_LOG(INFO, "Direct HW notifier FD cannot be "
-				"destroyed for device %d: %d.", priv->vid, ret);
-			return -1;
-		}
-		priv->direct_notifier = 0;
-	}
-	ret = rte_vhost_host_notifier_ctrl(priv->vid, RTE_VHOST_QUEUE_ALL,
-					   true);
-	if (ret != 0)
-		DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for"
-			" device %d: %d.", priv->vid, ret);
-	else
-		priv->direct_notifier = 1;
-	return 0;
-}
-
-static int
 mlx5_vdpa_features_set(int vid)
 {
 	int did = rte_vhost_get_vdpa_device_id(vid);
@@ -330,7 +305,6 @@
 	if (mlx5_vdpa_mtu_set(priv))
 		DRV_LOG(WARNING, "MTU cannot be set on device %d.", did);
 	if (mlx5_vdpa_pd_create(priv) || mlx5_vdpa_mem_register(priv) ||
-	    mlx5_vdpa_direct_db_prepare(priv) ||
 	    mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 	    mlx5_vdpa_cqe_event_setup(priv)) {
 		mlx5_vdpa_dev_close(vid);
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h
index 2ee5aae..8f349d4 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.h
@@ -73,11 +73,18 @@ struct mlx5_vdpa_query_mr {
 	int is_indirect;
 };
 
+enum {
+	MLX5_VDPA_NOTIFIER_STATE_DISABLED,
+	MLX5_VDPA_NOTIFIER_STATE_ENABLED,
+	MLX5_VDPA_NOTIFIER_STATE_ERR
+};
+
 struct mlx5_vdpa_virtq {
 	SLIST_ENTRY(mlx5_vdpa_virtq) next;
 	uint8_t enable;
 	uint16_t index;
 	uint16_t vq_size;
+	uint8_t notifier_state;
 	struct mlx5_vdpa_priv *priv;
 	struct mlx5_devx_obj *virtq;
 	struct mlx5_devx_obj *counters;
@@ -112,7 +119,6 @@ enum {
 struct mlx5_vdpa_priv {
 	TAILQ_ENTRY(mlx5_vdpa_priv) next;
 	uint8_t configured;
-	uint8_t direct_notifier; /* Whether direct notifier is on or off. */
 	uint64_t last_traffic_tic;
 	pthread_t timer_tid;
 	pthread_mutex_t timer_lock;
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
index 4b4d019..3e61264 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
@@ -36,6 +36,17 @@
 		break;
 	} while (1);
 	rte_write32(virtq->index, priv->virtq_db_addr);
+	if (virtq->notifier_state == MLX5_VDPA_NOTIFIER_STATE_DISABLED) {
+		if (rte_vhost_host_notifier_ctrl(priv->vid, virtq->index, true))
+			virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_ERR;
+		else
+			virtq->notifier_state =
+					       MLX5_VDPA_NOTIFIER_STATE_ENABLED;
+		DRV_LOG(INFO, "Virtq %u notifier state is %s.", virtq->index,
+			virtq->notifier_state ==
+				MLX5_VDPA_NOTIFIER_STATE_ENABLED ? "enabled" :
+								    "disabled");
+	}
 	DRV_LOG(DEBUG, "Ring virtq %u doorbell.", virtq->index);
 }
 
@@ -79,6 +90,7 @@
 	memset(&virtq->reset, 0, sizeof(virtq->reset));
 	if (virtq->eqp.fw_qp)
 		mlx5_vdpa_event_qp_destroy(&virtq->eqp);
+	virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_DISABLED;
 	return 0;
 }
 
@@ -87,10 +99,8 @@
 {
 	int i;
 
-	for (i = 0; i < priv->nr_virtqs; i++) {
+	for (i = 0; i < priv->nr_virtqs; i++)
 		mlx5_vdpa_virtq_unset(&priv->virtqs[i]);
-		priv->virtqs[i].enable = 0;
-	}
 	if (priv->tis) {
 		claim_zero(mlx5_devx_cmd_destroy(priv->tis));
 		priv->tis = NULL;
@@ -143,6 +153,7 @@
 		DRV_LOG(ERR, "Failed to set virtq %d base.", index);
 		return -1;
 	}
+	DRV_LOG(DEBUG, "vid %u virtq %u was stopped.", priv->vid, index);
 	return 0;
 }
 
@@ -289,6 +300,7 @@
 	virtq->priv = priv;
 	if (!virtq->virtq)
 		goto error;
+	claim_zero(rte_vhost_enable_guest_notification(priv->vid, index, 1));
 	if (mlx5_vdpa_virtq_modify(virtq, 1))
 		goto error;
 	virtq->priv = priv;
@@ -297,10 +309,6 @@
 	virtq->intr_handle.fd = vq.kickfd;
 	if (virtq->intr_handle.fd == -1) {
 		DRV_LOG(WARNING, "Virtq %d kickfd is invalid.", index);
-		if (!priv->direct_notifier) {
-			DRV_LOG(ERR, "Virtq %d cannot be notified.", index);
-			goto error;
-		}
 	} else {
 		virtq->intr_handle.type = RTE_INTR_HANDLE_EXT;
 		if (rte_intr_callback_register(&virtq->intr_handle,
@@ -315,6 +323,8 @@
 				virtq->intr_handle.fd, index);
 		}
 	}
+	DRV_LOG(DEBUG, "vid %u virtq %u was created successfully.", priv->vid,
+		index);
 	return 0;
 error:
 	mlx5_vdpa_virtq_unset(virtq);
@@ -418,18 +428,35 @@
 		goto error;
 	}
 	priv->nr_virtqs = nr_vring;
-	for (i = 0; i < nr_vring; i++) {
-		claim_zero(rte_vhost_enable_guest_notification(priv->vid, i,
-							       1));
-		if (mlx5_vdpa_virtq_setup(priv, i))
+	for (i = 0; i < nr_vring; i++)
+		if (priv->virtqs[i].enable && mlx5_vdpa_virtq_setup(priv, i))
 			goto error;
-	}
 	return 0;
 error:
 	mlx5_vdpa_virtqs_release(priv);
 	return -1;
 }
 
+static int
+mlx5_vdpa_virtq_is_modified(struct mlx5_vdpa_priv *priv,
+			    struct mlx5_vdpa_virtq *virtq)
+{
+	struct rte_vhost_vring vq;
+	int ret = rte_vhost_get_vhost_vring(priv->vid, virtq->index, &vq);
+
+	if (ret)
+		return -1;
+	if (vq.size != virtq->vq_size || vq.kickfd != virtq->intr_handle.fd)
+		return 1;
+	if (virtq->eqp.cq.cq) {
+		if (vq.callfd != virtq->eqp.cq.callfd)
+			return 1;
+	} else if (vq.callfd != -1) {
+		return 1;
+	}
+	return 0;
+}
+
 int
 mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable)
 {
@@ -438,26 +465,33 @@
 
 	DRV_LOG(INFO, "Update virtq %d status %sable -> %sable.", index,
 		virtq->enable ? "en" : "dis", enable ? "en" : "dis");
-	if (virtq->enable == !!enable)
-		return 0;
 	if (!priv->configured) {
 		virtq->enable = !!enable;
 		return 0;
 	}
-	if (enable) {
-		/* Configuration might have been updated - reconfigure virtq. */
-		if (virtq->virtq) {
-			ret = mlx5_vdpa_virtq_stop(priv, index);
-			if (ret)
-				DRV_LOG(WARNING, "Failed to stop virtq %d.",
-					index);
-			mlx5_vdpa_virtq_unset(virtq);
+	if (virtq->enable == !!enable) {
+		if (!enable)
+			return 0;
+		ret = mlx5_vdpa_virtq_is_modified(priv, virtq);
+		if (ret < 0) {
+			DRV_LOG(ERR, "Virtq %d modify check failed.", index);
+			return -1;
 		}
+		if (ret == 0)
+			return 0;
+		DRV_LOG(INFO, "Virtq %d was modified, recreate it.", index);
+	}
+	if (virtq->virtq) {
+		ret = mlx5_vdpa_virtq_stop(priv, index);
+		if (ret)
+			DRV_LOG(WARNING, "Failed to stop virtq %d.", index);
+		mlx5_vdpa_virtq_unset(virtq);
+	}
+	if (enable) {
 		ret = mlx5_vdpa_virtq_setup(priv, index);
 		if (ret) {
 			DRV_LOG(ERR, "Failed to setup virtq %d.", index);
 			return ret;
-			/* The only case virtq can stay invalid. */
 		}
 	}
 	virtq->enable = !!enable;
-- 
1.8.3.1


  parent reply	other threads:[~2020-06-29 14:09 UTC|newest]

Thread overview: 59+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-06-18 16:28 [dpdk-dev] [PATCH v1 0/4] vhost: improve ready state Matan Azrad
2020-06-18 16:28 ` [dpdk-dev] [PATCH v1 1/4] vhost: support host notifier queue configuration Matan Azrad
2020-06-19  6:44   ` Maxime Coquelin
2020-06-19 13:28     ` Matan Azrad
2020-06-19 14:01       ` Maxime Coquelin
2020-06-21  6:26         ` Matan Azrad
2020-06-22  8:06           ` Maxime Coquelin
2020-06-18 16:28 ` [dpdk-dev] [PATCH v1 2/4] vhost: skip access lock when vDPA is configured Matan Azrad
2020-06-19  6:49   ` Maxime Coquelin
2020-06-18 16:28 ` [dpdk-dev] [PATCH v1 3/4] vhost: improve device ready definition Matan Azrad
2020-06-19  7:41   ` Maxime Coquelin
2020-06-19 12:04     ` Maxime Coquelin
2020-06-19 13:11     ` Matan Azrad
2020-06-19 13:54       ` Maxime Coquelin
2020-06-21  6:20         ` Matan Azrad
2020-06-22  8:04           ` Maxime Coquelin
2020-06-22  8:41             ` Matan Azrad
2020-06-22  8:56               ` Maxime Coquelin
2020-06-22 10:06                 ` Matan Azrad
2020-06-22 12:32                   ` Maxime Coquelin
2020-06-22 13:43                     ` Matan Azrad
2020-06-22 14:55                       ` Maxime Coquelin
2020-06-22 15:51                         ` Matan Azrad
2020-06-22 16:47                           ` Maxime Coquelin
2020-06-23  9:02                             ` Matan Azrad
2020-06-23  9:19                               ` Maxime Coquelin
2020-06-23 11:53                                 ` Matan Azrad
2020-06-23 13:55                                   ` Maxime Coquelin
2020-06-23 14:33                                     ` Maxime Coquelin
2020-06-23 14:52                                     ` Matan Azrad
2020-06-23 15:18                                       ` Maxime Coquelin
2020-06-24  5:54                                         ` Matan Azrad
2020-06-24  7:22                                           ` Maxime Coquelin
2020-06-24  8:38                                             ` Matan Azrad
2020-06-24  9:12                                               ` Maxime Coquelin
2020-06-18 16:28 ` [dpdk-dev] [PATCH v1 4/4] vdpa/mlx5: support queue update Matan Azrad
2020-06-25 13:38 ` [dpdk-dev] [PATCH v2 0/5] vhost: improve ready state Matan Azrad
2020-06-25 13:38   ` [dpdk-dev] [PATCH v2 1/5] vhost: skip access lock when vDPA is configured Matan Azrad
2020-06-28  3:06     ` Xia, Chenbo
2020-06-25 13:38   ` [dpdk-dev] [PATCH v2 2/5] vhost: improve device readiness notifications Matan Azrad
2020-06-26 12:10     ` Maxime Coquelin
2020-06-28  3:08     ` Xia, Chenbo
2020-06-25 13:38   ` [dpdk-dev] [PATCH v2 3/5] vhost: handle memory hotplug with vDPA devices Matan Azrad
2020-06-26 12:15     ` Maxime Coquelin
2020-06-28  3:18     ` Xia, Chenbo
2020-06-25 13:38   ` [dpdk-dev] [PATCH v2 4/5] vhost: notify virtq file descriptor update Matan Azrad
2020-06-26 12:19     ` Maxime Coquelin
2020-06-28  3:19     ` Xia, Chenbo
2020-06-25 13:38   ` [dpdk-dev] [PATCH v2 5/5] vdpa/mlx5: support queue update Matan Azrad
2020-06-26 12:29     ` Maxime Coquelin
2020-06-29 14:08   ` [dpdk-dev] [PATCH v3 0/6] vhost: improve ready state Matan Azrad
2020-06-29 14:08     ` [dpdk-dev] [PATCH v3 1/6] vhost: support host notifier queue configuration Matan Azrad
2020-06-29 14:08     ` [dpdk-dev] [PATCH v3 2/6] vhost: skip access lock when vDPA is configured Matan Azrad
2020-06-29 14:08     ` [dpdk-dev] [PATCH v3 3/6] vhost: improve device readiness notifications Matan Azrad
2020-06-29 14:08     ` [dpdk-dev] [PATCH v3 4/6] vhost: handle memory hotplug with vDPA devices Matan Azrad
2020-06-29 14:08     ` [dpdk-dev] [PATCH v3 5/6] vhost: notify virtq file descriptor update Matan Azrad
2020-06-29 14:08     ` Matan Azrad [this message]
2020-06-29 17:24     ` [dpdk-dev] [PATCH v3 0/6] vhost: improve ready state Maxime Coquelin
2020-07-17  1:41       ` Wang, Yinan

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1593439701-136089-7-git-send-email-matan@mellanox.com \
    --to=matan@mellanox.com \
    --cc=dev@dpdk.org \
    --cc=maxime.coquelin@redhat.com \
    --cc=xiao.w.wang@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

DPDK patches and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git