[PATCH v5 15/26] vhost: add control virtqueue support

[Maxime Coquelin <maxime.coquelin@redhat.com>]

In order to support multi-queue with VDUSE, having
control queue support is required.

This patch adds control queue implementation, it will be
used later when adding VDUSE support. Only split ring
layout is supported for now, packed ring support will be
added later.

Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
---
 lib/vhost/meson.build       |   1 +
 lib/vhost/vhost.h           |   2 +
 lib/vhost/virtio_net_ctrl.c | 286 ++++++++++++++++++++++++++++++++++++
 lib/vhost/virtio_net_ctrl.h |  10 ++
 4 files changed, 299 insertions(+)
 create mode 100644 lib/vhost/virtio_net_ctrl.c
 create mode 100644 lib/vhost/virtio_net_ctrl.h

diff --git a/lib/vhost/meson.build b/lib/vhost/meson.build
index 05679447db..d211a0bd37 100644
--- a/lib/vhost/meson.build
+++ b/lib/vhost/meson.build
@@ -27,6 +27,7 @@ sources = files(
         'vhost_crypto.c',
         'vhost_user.c',
         'virtio_net.c',
+        'virtio_net_ctrl.c',
 )
 headers = files(
         'rte_vdpa.h',
diff --git a/lib/vhost/vhost.h b/lib/vhost/vhost.h
index ea2798b0bf..04267a3ac5 100644
--- a/lib/vhost/vhost.h
+++ b/lib/vhost/vhost.h
@@ -527,6 +527,8 @@ struct virtio_net {
 	int			postcopy_ufd;
 	int			postcopy_listening;
 
+	struct vhost_virtqueue	*cvq;
+
 	struct rte_vdpa_device *vdpa_dev;
 
 	/* context data for the external message handlers */
diff --git a/lib/vhost/virtio_net_ctrl.c b/lib/vhost/virtio_net_ctrl.c
new file mode 100644
index 0000000000..6b583a0ce6
--- /dev/null
+++ b/lib/vhost/virtio_net_ctrl.c
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Red Hat, Inc.
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include "iotlb.h"
+#include "vhost.h"
+#include "virtio_net_ctrl.h"
+
+struct virtio_net_ctrl {
+	uint8_t class;
+	uint8_t command;
+	uint8_t command_data[];
+};
+
+struct virtio_net_ctrl_elem {
+	struct virtio_net_ctrl *ctrl_req;
+	uint16_t head_idx;
+	uint16_t n_descs;
+	uint8_t *desc_ack;
+};
+
+static int
+virtio_net_ctrl_pop(struct virtio_net *dev, struct vhost_virtqueue *cvq,
+		struct virtio_net_ctrl_elem *ctrl_elem)
+	__rte_shared_locks_required(&cvq->iotlb_lock)
+{
+	uint16_t avail_idx, desc_idx, n_descs = 0;
+	uint64_t desc_len, desc_addr, desc_iova, data_len = 0;
+	uint8_t *ctrl_req;
+	struct vring_desc *descs;
+
+	avail_idx = __atomic_load_n(&cvq->avail->idx, __ATOMIC_ACQUIRE);
+	if (avail_idx == cvq->last_avail_idx) {
+		VHOST_LOG_CONFIG(dev->ifname, DEBUG, "Control queue empty\n");
+		return 0;
+	}
+
+	desc_idx = cvq->avail->ring[cvq->last_avail_idx];
+	if (desc_idx >= cvq->size) {
+		VHOST_LOG_CONFIG(dev->ifname, ERR, "Out of range desc index, dropping\n");
+		goto err;
+	}
+
+	ctrl_elem->head_idx = desc_idx;
+
+	if (cvq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT) {
+		desc_len = cvq->desc[desc_idx].len;
+		desc_iova = cvq->desc[desc_idx].addr;
+
+		descs = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, cvq,
+					desc_iova, &desc_len, VHOST_ACCESS_RO);
+		if (!descs || desc_len != cvq->desc[desc_idx].len) {
+			VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to map ctrl indirect descs\n");
+			goto err;
+		}
+
+		desc_idx = 0;
+	} else {
+		descs = cvq->desc;
+	}
+
+	while (1) {
+		desc_len = descs[desc_idx].len;
+		desc_iova = descs[desc_idx].addr;
+
+		n_descs++;
+
+		if (descs[desc_idx].flags & VRING_DESC_F_WRITE) {
+			if (ctrl_elem->desc_ack) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+						"Unexpected ctrl chain layout\n");
+				goto err;
+			}
+
+			if (desc_len != sizeof(uint8_t)) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+						"Invalid ack size for ctrl req, dropping\n");
+				goto err;
+			}
+
+			ctrl_elem->desc_ack = (uint8_t *)(uintptr_t)vhost_iova_to_vva(dev, cvq,
+					desc_iova, &desc_len, VHOST_ACCESS_WO);
+			if (!ctrl_elem->desc_ack || desc_len != sizeof(uint8_t)) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+						"Failed to map ctrl ack descriptor\n");
+				goto err;
+			}
+		} else {
+			if (ctrl_elem->desc_ack) {
+				VHOST_LOG_CONFIG(dev->ifname, ERR,
+						"Unexpected ctrl chain layout\n");
+				goto err;
+			}
+
+			data_len += desc_len;
+		}
+
+		if (!(descs[desc_idx].flags & VRING_DESC_F_NEXT))
+			break;
+
+		desc_idx = descs[desc_idx].next;
+	}
+
+	desc_idx = ctrl_elem->head_idx;
+
+	if (cvq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT)
+		ctrl_elem->n_descs = 1;
+	else
+		ctrl_elem->n_descs = n_descs;
+
+	if (!ctrl_elem->desc_ack) {
+		VHOST_LOG_CONFIG(dev->ifname, ERR, "Missing ctrl ack descriptor\n");
+		goto err;
+	}
+
+	if (data_len < sizeof(ctrl_elem->ctrl_req->class) + sizeof(ctrl_elem->ctrl_req->command)) {
+		VHOST_LOG_CONFIG(dev->ifname, ERR, "Invalid control header size\n");
+		goto err;
+	}
+
+	ctrl_elem->ctrl_req = malloc(data_len);
+	if (!ctrl_elem->ctrl_req) {
+		VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to alloc ctrl request\n");
+		goto err;
+	}
+
+	ctrl_req = (uint8_t *)ctrl_elem->ctrl_req;
+
+	if (cvq->desc[desc_idx].flags & VRING_DESC_F_INDIRECT) {
+		desc_len = cvq->desc[desc_idx].len;
+		desc_iova = cvq->desc[desc_idx].addr;
+
+		descs = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, cvq,
+					desc_iova, &desc_len, VHOST_ACCESS_RO);
+		if (!descs || desc_len != cvq->desc[desc_idx].len) {
+			VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to map ctrl indirect descs\n");
+			goto free_err;
+		}
+
+		desc_idx = 0;
+	} else {
+		descs = cvq->desc;
+	}
+
+	while (!(descs[desc_idx].flags & VRING_DESC_F_WRITE)) {
+		desc_len = descs[desc_idx].len;
+		desc_iova = descs[desc_idx].addr;
+
+		desc_addr = vhost_iova_to_vva(dev, cvq, desc_iova, &desc_len, VHOST_ACCESS_RO);
+		if (!desc_addr || desc_len < descs[desc_idx].len) {
+			VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to map ctrl descriptor\n");
+			goto free_err;
+		}
+
+		memcpy(ctrl_req, (void *)(uintptr_t)desc_addr, desc_len);
+		ctrl_req += desc_len;
+
+		if (!(descs[desc_idx].flags & VRING_DESC_F_NEXT))
+			break;
+
+		desc_idx = descs[desc_idx].next;
+	}
+
+	cvq->last_avail_idx++;
+	if (cvq->last_avail_idx >= cvq->size)
+		cvq->last_avail_idx -= cvq->size;
+
+	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
+		vhost_avail_event(cvq) = cvq->last_avail_idx;
+
+	return 1;
+
+free_err:
+	free(ctrl_elem->ctrl_req);
+err:
+	cvq->last_avail_idx++;
+	if (cvq->last_avail_idx >= cvq->size)
+		cvq->last_avail_idx -= cvq->size;
+
+	if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
+		vhost_avail_event(cvq) = cvq->last_avail_idx;
+
+	return -1;
+}
+
+static uint8_t
+virtio_net_ctrl_handle_req(struct virtio_net *dev, struct virtio_net_ctrl *ctrl_req)
+{
+	uint8_t ret = VIRTIO_NET_ERR;
+
+	if (ctrl_req->class == VIRTIO_NET_CTRL_MQ &&
+			ctrl_req->command == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
+		uint16_t queue_pairs;
+		uint32_t i;
+
+		queue_pairs = *(uint16_t *)(uintptr_t)ctrl_req->command_data;
+		VHOST_LOG_CONFIG(dev->ifname, INFO, "Ctrl req: MQ %u queue pairs\n", queue_pairs);
+		ret = VIRTIO_NET_OK;
+
+		for (i = 0; i < dev->nr_vring; i++) {
+			struct vhost_virtqueue *vq = dev->virtqueue[i];
+			bool enable;
+
+			if (vq == dev->cvq)
+				continue;
+
+			if (i < queue_pairs * 2)
+				enable = true;
+			else
+				enable = false;
+
+			vq->enabled = enable;
+			if (dev->notify_ops->vring_state_changed)
+				dev->notify_ops->vring_state_changed(dev->vid, i, enable);
+		}
+	}
+
+	return ret;
+}
+
+static int
+virtio_net_ctrl_push(struct virtio_net *dev, struct virtio_net_ctrl_elem *ctrl_elem)
+{
+	struct vhost_virtqueue *cvq = dev->cvq;
+	struct vring_used_elem *used_elem;
+
+	used_elem = &cvq->used->ring[cvq->last_used_idx];
+	used_elem->id = ctrl_elem->head_idx;
+	used_elem->len = ctrl_elem->n_descs;
+
+	cvq->last_used_idx++;
+	if (cvq->last_used_idx >= cvq->size)
+		cvq->last_used_idx -= cvq->size;
+
+	__atomic_store_n(&cvq->used->idx, cvq->last_used_idx, __ATOMIC_RELEASE);
+
+	vhost_vring_call_split(dev, dev->cvq);
+
+	free(ctrl_elem->ctrl_req);
+
+	return 0;
+}
+
+int
+virtio_net_ctrl_handle(struct virtio_net *dev)
+{
+	int ret = 0;
+
+	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
+		VHOST_LOG_CONFIG(dev->ifname, ERR, "Packed ring not supported yet\n");
+		return -1;
+	}
+
+	if (!dev->cvq) {
+		VHOST_LOG_CONFIG(dev->ifname, ERR, "missing control queue\n");
+		return -1;
+	}
+
+	rte_rwlock_read_lock(&dev->cvq->access_lock);
+	vhost_user_iotlb_rd_lock(dev->cvq);
+
+	while (1) {
+		struct virtio_net_ctrl_elem ctrl_elem;
+
+		memset(&ctrl_elem, 0, sizeof(struct virtio_net_ctrl_elem));
+
+		ret = virtio_net_ctrl_pop(dev, dev->cvq, &ctrl_elem);
+		if (ret <= 0)
+			break;
+
+		*ctrl_elem.desc_ack = virtio_net_ctrl_handle_req(dev, ctrl_elem.ctrl_req);
+
+		ret = virtio_net_ctrl_push(dev, &ctrl_elem);
+		if (ret < 0)
+			break;
+	}
+
+	vhost_user_iotlb_rd_unlock(dev->cvq);
+	rte_rwlock_read_unlock(&dev->cvq->access_lock);
+
+	return ret;
+}
diff --git a/lib/vhost/virtio_net_ctrl.h b/lib/vhost/virtio_net_ctrl.h
new file mode 100644
index 0000000000..9a90f4b9da
--- /dev/null
+++ b/lib/vhost/virtio_net_ctrl.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 Red Hat, Inc.
+ */
+
+#ifndef _VIRTIO_NET_CTRL_H
+#define _VIRTIO_NET_CTRL_H
+
+int virtio_net_ctrl_handle(struct virtio_net *dev);
+
+#endif
-- 
2.40.1