patch 'vdpa/sfc: resolve race between vhost lib and device conf' has been queued to stable release 21.11.2

[luca.boccassi@gmail.com]

Hi,

FYI, your patch has been queued to stable release 21.11.2

Note it hasn't been pushed to http://dpdk.org/browse/dpdk-stable yet.
It will be pushed if I get no objections before 07/14/22. So please
shout if anyone has objections.

Also note that after the patch there's a diff of the upstream commit vs the
patch applied to the branch. This will indicate if there was any rebasing
needed to apply to the stable branch. If there were code changes for rebasing
(ie: not only metadata diffs), please double check that the rebase was
correctly done.

Queued patches are on a temporary branch at:
https://github.com/kevintraynor/dpdk-stable

This queued commit can be viewed at:
https://github.com/kevintraynor/dpdk-stable/commit/b11e955370a713ee418ee0f5e853364fe72132a8

Thanks.

Luca Boccassi

---
From b11e955370a713ee418ee0f5e853364fe72132a8 Mon Sep 17 00:00:00 2001
From: Abhimanyu Saini <absaini@amd.com>
Date: Wed, 6 Jul 2022 14:54:01 +0530
Subject: [PATCH] vdpa/sfc: resolve race between vhost lib and device conf

[ upstream commit d9a1ef20f0f1c6f4029f79d5eab5dc696d115c81 ]

libvhost calls dev_conf() before prosessing the
VHOST_USER_SET_VRING_CALL message for the last VQ. So
this message is processed after dev_conf() returns.

However, the dev_conf() function spawns a thread to set
rte_vhost_host_notifier_ctrl() before returning control to
libvhost. This parallel thread in turn invokes get_notify_area().
To get the notify_area, the vdpa driver needs to query the HW and
for this query it needs an enabled VQ.

But at the same time libvhost is processing the last
VHOST_USER_SET_VRING_CALL, and to do that it disables the last VQ.

Hence there is a race b/w the libvhost and the vdpa driver.

To resolve this race condition, query the HW and cache notify_area
inside dev_conf() instead of doing it the parallel thread.

Fixes: 630be406dcbf ("vdpa/sfc: get queue notify area info")

Signed-off-by: Abhimanyu Saini <absaini@amd.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 36 ++++++++++++++++-----------------
 drivers/vdpa/sfc/sfc_vdpa_ops.h |  1 +
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/drivers/vdpa/sfc/sfc_vdpa_ops.c b/drivers/vdpa/sfc/sfc_vdpa_ops.c
index 63aa52da49..b84699d234 100644
--- a/drivers/vdpa/sfc/sfc_vdpa_ops.c
+++ b/drivers/vdpa/sfc/sfc_vdpa_ops.c
@@ -222,6 +222,7 @@ static int
 sfc_vdpa_virtq_start(struct sfc_vdpa_ops_data *ops_data, int vq_num)
 {
 	int rc;
+	uint32_t doorbell;
 	efx_virtio_vq_t *vq;
 	struct sfc_vdpa_vring_info vring;
 	efx_virtio_vq_cfg_t vq_cfg;
@@ -270,22 +271,35 @@ sfc_vdpa_virtq_start(struct sfc_vdpa_ops_data *ops_data, int vq_num)
 	/* Start virtqueue */
 	rc = efx_virtio_qstart(vq, &vq_cfg, &vq_dyncfg);
 	if (rc != 0) {
-		/* destroy virtqueue */
 		sfc_vdpa_err(ops_data->dev_handle,
 			     "virtqueue start failed: %s",
 			     rte_strerror(rc));
-		efx_virtio_qdestroy(vq);
 		goto fail_virtio_qstart;
 	}
 
 	sfc_vdpa_info(ops_data->dev_handle,
 		      "virtqueue started successfully for vq_num %d", vq_num);
 
+	rc = efx_virtio_get_doorbell_offset(vq,	&doorbell);
+	if (rc != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "failed to get doorbell offset: %s",
+			     rte_strerror(rc));
+		goto fail_doorbell;
+	}
+
+	/*
+	 * Cache the bar_offset here for each VQ here, it will come
+	 * in handy when sfc_vdpa_get_notify_area() is invoked.
+	 */
+	ops_data->vq_cxt[vq_num].doorbell = (void *)(uintptr_t)doorbell;
 	ops_data->vq_cxt[vq_num].enable = B_TRUE;
 
 	return rc;
 
+fail_doorbell:
 fail_virtio_qstart:
+	efx_virtio_qdestroy(vq);
 fail_vring_info:
 	return rc;
 }
@@ -792,8 +806,6 @@ sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 	int ret;
 	efx_nic_t *nic;
 	int vfio_dev_fd;
-	efx_rc_t rc;
-	unsigned int bar_offset;
 	volatile void *doorbell;
 	struct rte_pci_device *pci_dev;
 	struct rte_vdpa_device *vdpa_dev;
@@ -824,19 +836,6 @@ sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 		return -1;
 	}
 
-	if (ops_data->vq_cxt[qid].enable != B_TRUE) {
-		sfc_vdpa_err(dev, "vq is not enabled");
-		return -1;
-	}
-
-	rc = efx_virtio_get_doorbell_offset(ops_data->vq_cxt[qid].vq,
-					    &bar_offset);
-	if (rc != 0) {
-		sfc_vdpa_err(dev, "failed to get doorbell offset: %s",
-			     rte_strerror(rc));
-		return rc;
-	}
-
 	reg.index = sfc_vdpa_adapter_by_dev_handle(dev)->mem_bar.esb_rid;
 	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg);
 	if (ret != 0) {
@@ -845,7 +844,8 @@ sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
 		return ret;
 	}
 
-	*offset = reg.offset + bar_offset;
+	/* Use bar_offset that was cached during sfc_vdpa_virtq_start() */
+	*offset = reg.offset + (uint64_t)ops_data->vq_cxt[qid].doorbell;
 
 	len = (1U << encp->enc_vi_window_shift) / 2;
 	if (len >= sysconf(_SC_PAGESIZE)) {
diff --git a/drivers/vdpa/sfc/sfc_vdpa_ops.h b/drivers/vdpa/sfc/sfc_vdpa_ops.h
index 6d790fd8be..9dbd5b84dd 100644
--- a/drivers/vdpa/sfc/sfc_vdpa_ops.h
+++ b/drivers/vdpa/sfc/sfc_vdpa_ops.h
@@ -35,6 +35,7 @@ struct sfc_vdpa_vring_info {
 };
 
 typedef struct sfc_vdpa_vq_context_s {
+	volatile void			*doorbell;
 	uint8_t				enable;
 	uint32_t			pidx;
 	uint32_t			cidx;
-- 
2.34.1

---
  Diff of the applied patch vs upstream commit (please double-check if non-empty:
---
--- -	2022-07-12 20:22:33.849529911 +0100
+++ 0004-vdpa-sfc-resolve-race-between-vhost-lib-and-device-c.patch	2022-07-12 20:22:33.713247244 +0100
@@ -1 +1 @@
-From d9a1ef20f0f1c6f4029f79d5eab5dc696d115c81 Mon Sep 17 00:00:00 2001
+From b11e955370a713ee418ee0f5e853364fe72132a8 Mon Sep 17 00:00:00 2001
@@ -5,0 +6,2 @@
+[ upstream commit d9a1ef20f0f1c6f4029f79d5eab5dc696d115c81 ]
+
@@ -25 +26,0 @@
-Cc: stable@dpdk.org