From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id C52E2A04F1; Thu, 18 Jun 2020 18:28:51 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 6D5131BFD1; Thu, 18 Jun 2020 18:28:21 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id C30E41BFAA for ; Thu, 18 Jun 2020 18:28:13 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from matan@mellanox.com) with SMTP; 18 Jun 2020 19:28:10 +0300 Received: from pegasus25.mtr.labs.mlnx. (pegasus25.mtr.labs.mlnx [10.210.16.10]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 05IGS9ER008363; Thu, 18 Jun 2020 19:28:10 +0300 From: Matan Azrad To: Maxime Coquelin , Xiao Wang Cc: dev@dpdk.org Date: Thu, 18 Jun 2020 16:28:06 +0000 Message-Id: <1592497686-433697-5-git-send-email-matan@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592497686-433697-1-git-send-email-matan@mellanox.com> References: <1592497686-433697-1-git-send-email-matan@mellanox.com> Subject: [dpdk-dev] [PATCH v1 4/4] vdpa/mlx5: support queue update X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Last changes in vDPA device management by vhost library may cause queue ready state update after the device configuration. So, there is chance that some queue configuration information will be known only after the device was configured. Add support to reconfigure a queue after the device configuration according to the queue state update and the configuration changes. Adjust the host notifier and the guest notification configuration to be per queue and to be applied in the enablement process. Signed-off-by: Matan Azrad --- drivers/vdpa/mlx5/mlx5_vdpa.c | 25 ---------------- drivers/vdpa/mlx5/mlx5_vdpa.h | 8 ++++- drivers/vdpa/mlx5/mlx5_vdpa_virtq.c | 58 ++++++++++++++++++++++++++++++------- 3 files changed, 54 insertions(+), 37 deletions(-) diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c index 8ea1300..0ef9e85 100644 --- a/drivers/vdpa/mlx5/mlx5_vdpa.c +++ b/drivers/vdpa/mlx5/mlx5_vdpa.c @@ -142,30 +142,6 @@ } static int -mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv) -{ - int ret; - - if (priv->direct_notifier) { - ret = rte_vhost_host_notifier_ctrl(priv->vid, VHOST_QUEUE_ALL, - false); - if (ret != 0) { - DRV_LOG(INFO, "Direct HW notifier FD cannot be " - "destroyed for device %d: %d.", priv->vid, ret); - return -1; - } - priv->direct_notifier = 0; - } - ret = rte_vhost_host_notifier_ctrl(priv->vid, VHOST_QUEUE_ALL, true); - if (ret != 0) - DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for" - " device %d: %d.", priv->vid, ret); - else - priv->direct_notifier = 1; - return 0; -} - -static int mlx5_vdpa_features_set(int vid) { int did = rte_vhost_get_vdpa_device_id(vid); @@ -330,7 +306,6 @@ if (mlx5_vdpa_mtu_set(priv)) DRV_LOG(WARNING, "MTU cannot be set on device %d.", did); if (mlx5_vdpa_pd_create(priv) || mlx5_vdpa_mem_register(priv) || - mlx5_vdpa_direct_db_prepare(priv) || mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) || mlx5_vdpa_cqe_event_setup(priv)) { mlx5_vdpa_dev_close(vid); diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h index 28ec0be..0b90900 100644 --- a/drivers/vdpa/mlx5/mlx5_vdpa.h +++ b/drivers/vdpa/mlx5/mlx5_vdpa.h @@ -70,11 +70,18 @@ struct mlx5_vdpa_query_mr { int is_indirect; }; +enum { + MLX5_VDPA_NOTIFIER_STATE_DISABLED, + MLX5_VDPA_NOTIFIER_STATE_ENABLED, + MLX5_VDPA_NOTIFIER_STATE_ERR +}; + struct mlx5_vdpa_virtq { SLIST_ENTRY(mlx5_vdpa_virtq) next; uint8_t enable; uint16_t index; uint16_t vq_size; + uint8_t notifier_state; struct mlx5_vdpa_priv *priv; struct mlx5_devx_obj *virtq; struct mlx5_devx_obj *counters; @@ -103,7 +110,6 @@ struct mlx5_vdpa_steer { struct mlx5_vdpa_priv { TAILQ_ENTRY(mlx5_vdpa_priv) next; uint8_t configured; - uint8_t direct_notifier; /* Whether direct notifier is on or off. */ uint64_t last_traffic_tic; pthread_t timer_tid; pthread_mutex_t timer_lock; diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c index 4b4d019..30d45d4 100644 --- a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c +++ b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c @@ -36,6 +36,17 @@ break; } while (1); rte_write32(virtq->index, priv->virtq_db_addr); + if (virtq->notifier_state == MLX5_VDPA_NOTIFIER_STATE_DISABLED) { + if (rte_vhost_host_notifier_ctrl(priv->vid, virtq->index, true)) + virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_ERR; + else + virtq->notifier_state = + MLX5_VDPA_NOTIFIER_STATE_ENABLED; + DRV_LOG(INFO, "Virtq %u notifier state is %s.", virtq->index, + virtq->notifier_state == + MLX5_VDPA_NOTIFIER_STATE_ENABLED ? "enabled" : + "disabled"); + } DRV_LOG(DEBUG, "Ring virtq %u doorbell.", virtq->index); } @@ -79,6 +90,7 @@ memset(&virtq->reset, 0, sizeof(virtq->reset)); if (virtq->eqp.fw_qp) mlx5_vdpa_event_qp_destroy(&virtq->eqp); + virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_DISABLED; return 0; } @@ -289,6 +301,7 @@ virtq->priv = priv; if (!virtq->virtq) goto error; + claim_zero(rte_vhost_enable_guest_notification(priv->vid, index, 1)); if (mlx5_vdpa_virtq_modify(virtq, 1)) goto error; virtq->priv = priv; @@ -297,10 +310,6 @@ virtq->intr_handle.fd = vq.kickfd; if (virtq->intr_handle.fd == -1) { DRV_LOG(WARNING, "Virtq %d kickfd is invalid.", index); - if (!priv->direct_notifier) { - DRV_LOG(ERR, "Virtq %d cannot be notified.", index); - goto error; - } } else { virtq->intr_handle.type = RTE_INTR_HANDLE_EXT; if (rte_intr_callback_register(&virtq->intr_handle, @@ -418,18 +427,35 @@ goto error; } priv->nr_virtqs = nr_vring; - for (i = 0; i < nr_vring; i++) { - claim_zero(rte_vhost_enable_guest_notification(priv->vid, i, - 1)); - if (mlx5_vdpa_virtq_setup(priv, i)) + for (i = 0; i < nr_vring; i++) + if (priv->virtqs[i].enable && mlx5_vdpa_virtq_setup(priv, i)) goto error; - } return 0; error: mlx5_vdpa_virtqs_release(priv); return -1; } +static int +mlx5_vdpa_virtq_is_modified(struct mlx5_vdpa_priv *priv, + struct mlx5_vdpa_virtq *virtq) +{ + struct rte_vhost_vring vq; + int ret = rte_vhost_get_vhost_vring(priv->vid, virtq->index, &vq); + + if (ret) + return -1; + if (vq.size != virtq->vq_size || vq.kickfd != virtq->intr_handle.fd) + return 1; + if (virtq->eqp.cq.cq) { + if (vq.callfd != virtq->eqp.cq.callfd) + return 1; + } else if (vq.callfd != -1) { + return 1; + } + return 0; +} + int mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable) { @@ -438,12 +464,22 @@ DRV_LOG(INFO, "Update virtq %d status %sable -> %sable.", index, virtq->enable ? "en" : "dis", enable ? "en" : "dis"); - if (virtq->enable == !!enable) - return 0; if (!priv->configured) { virtq->enable = !!enable; return 0; } + if (virtq->enable == !!enable) { + if (!enable) + return 0; + ret = mlx5_vdpa_virtq_is_modified(priv, virtq); + if (ret < 0) { + DRV_LOG(ERR, "Virtq %d modify check failed.", index); + return -1; + } + if (ret == 0) + return 0; + DRV_LOG(INFO, "Virtq %d was modified, recreate it.", index); + } if (enable) { /* Configuration might have been updated - reconfigure virtq. */ if (virtq->virtq) { -- 1.8.3.1