From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 96E1EE72 for ; Mon, 21 Sep 2015 11:07:23 +0200 (CEST) Received: from int-mx11.intmail.prod.int.phx2.redhat.com (int-mx11.intmail.prod.int.phx2.redhat.com [10.5.11.24]) by mx1.redhat.com (Postfix) with ESMTPS id 15B6DC0A1465; Mon, 21 Sep 2015 09:07:23 +0000 (UTC) Received: from redhat.com (ovpn-116-114.ams2.redhat.com [10.36.116.114]) by int-mx11.intmail.prod.int.phx2.redhat.com (8.14.4/8.14.4) with SMTP id t8L97KT5027930; Mon, 21 Sep 2015 05:07:21 -0400 Date: Mon, 21 Sep 2015 12:07:20 +0300 From: "Michael S. Tsirkin" To: marcel@redhat.com Message-ID: <20150921120459-mutt-send-email-mst@redhat.com> References: <1442589061-19225-1-git-send-email-yuanhan.liu@linux.intel.com> <1442589061-19225-4-git-send-email-yuanhan.liu@linux.intel.com> <55FEBB92.40403@gmail.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <55FEBB92.40403@gmail.com> X-Scanned-By: MIMEDefang 2.68 on 10.5.11.24 Cc: dev@dpdk.org Subject: Re: [dpdk-dev] [PATCH v5 resend 03/12] vhost: vring queue setup for multiple queue support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 21 Sep 2015 09:07:24 -0000 On Sun, Sep 20, 2015 at 04:58:42PM +0300, Marcel Apfelbaum wrote: > On 09/18/2015 06:10 PM, Yuanhan Liu wrote: > >All queue pairs, including the default (the first) queue pair, > >are allocated dynamically, when a vring_call message is received > >first time for a specific queue pair. > > > >This is a refactor work for enabling vhost-user multiple queue; > >it should not break anything as it does no functional changes: > >we don't support mq set, so there is only one mq at max. > > > >This patch is based on Changchun's patch. > > > >Signed-off-by: Yuanhan Liu > >--- > > lib/librte_vhost/rte_virtio_net.h | 3 +- > > lib/librte_vhost/vhost_user/virtio-net-user.c | 44 +++++----- > > lib/librte_vhost/virtio-net.c | 121 ++++++++++++++++---------- > > 3 files changed, 102 insertions(+), 66 deletions(-) > > > >diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h > >index e3a21e5..5dd6493 100644 > >--- a/lib/librte_vhost/rte_virtio_net.h > >+++ b/lib/librte_vhost/rte_virtio_net.h > >@@ -96,7 +96,7 @@ struct vhost_virtqueue { > > * Device structure contains all configuration information relating to the device. > > */ > > struct virtio_net { > >- struct vhost_virtqueue *virtqueue[VIRTIO_QNUM]; /**< Contains all virtqueue information. */ > >+ struct vhost_virtqueue *virtqueue[VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX]; /**< Contains all virtqueue information. */ > > struct virtio_memory *mem; /**< QEMU memory and memory region information. */ > > uint64_t features; /**< Negotiated feature set. */ > > uint64_t protocol_features; /**< Negotiated protocol feature set. */ > >@@ -104,6 +104,7 @@ struct virtio_net { > > uint32_t flags; /**< Device flags. Only used to check if device is running on data core. */ > > #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ) > > char ifname[IF_NAME_SZ]; /**< Name of the tap device or socket path. */ > >+ uint32_t virt_qp_nb; /**< number of queue pair we have allocated */ > > void *priv; /**< private context */ > > } __rte_cache_aligned; > > > >diff --git a/lib/librte_vhost/vhost_user/virtio-net-user.c b/lib/librte_vhost/vhost_user/virtio-net-user.c > >index 360254e..e83d279 100644 > >--- a/lib/librte_vhost/vhost_user/virtio-net-user.c > >+++ b/lib/librte_vhost/vhost_user/virtio-net-user.c > >@@ -206,25 +206,33 @@ err_mmap: > > } > > > > Hi, > > > static int > >+vq_is_ready(struct vhost_virtqueue *vq) > >+{ > >+ return vq && vq->desc && > >+ vq->kickfd != -1 && > >+ vq->callfd != -1; > > kickfd and callfd are unsigned That's probably a bug. fds are signed, and -1 is what qemu uses to mean "nop". This comparison will convert -1 to unsigned int so it'll work. The >= ones below won't work. I think fd types need to be fixed. > >+} > >+ > >+static int > > virtio_is_ready(struct virtio_net *dev) > > { > > struct vhost_virtqueue *rvq, *tvq; > >+ uint32_t i; > > > >- /* mq support in future.*/ > >- rvq = dev->virtqueue[VIRTIO_RXQ]; > >- tvq = dev->virtqueue[VIRTIO_TXQ]; > >- if (rvq && tvq && rvq->desc && tvq->desc && > >- (rvq->kickfd != -1) && > >- (rvq->callfd != -1) && > >- (tvq->kickfd != -1) && > >- (tvq->callfd != -1)) { > >- RTE_LOG(INFO, VHOST_CONFIG, > >- "virtio is now ready for processing.\n"); > >- return 1; > >+ for (i = 0; i < dev->virt_qp_nb; i++) { > >+ rvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ]; > >+ tvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ]; > >+ > >+ if (!vq_is_ready(rvq) || !vq_is_ready(tvq)) { > >+ RTE_LOG(INFO, VHOST_CONFIG, > >+ "virtio is not ready for processing.\n"); > >+ return 0; > >+ } > > } > >+ > > RTE_LOG(INFO, VHOST_CONFIG, > >- "virtio isn't ready for processing.\n"); > >- return 0; > >+ "virtio is now ready for processing.\n"); > >+ return 1; > > } > > > > void > >@@ -290,13 +298,9 @@ user_get_vring_base(struct vhost_device_ctx ctx, > > * sent and only sent in vhost_vring_stop. > > * TODO: cleanup the vring, it isn't usable since here. > > */ > >- if ((dev->virtqueue[VIRTIO_RXQ]->kickfd) >= 0) { > >- close(dev->virtqueue[VIRTIO_RXQ]->kickfd); > >- dev->virtqueue[VIRTIO_RXQ]->kickfd = -1; > >- } > >- if ((dev->virtqueue[VIRTIO_TXQ]->kickfd) >= 0) { > >- close(dev->virtqueue[VIRTIO_TXQ]->kickfd); > >- dev->virtqueue[VIRTIO_TXQ]->kickfd = -1; > >+ if ((dev->virtqueue[state->index]->kickfd) >= 0) { > > always >= 0 > > >+ close(dev->virtqueue[state->index]->kickfd); > >+ dev->virtqueue[state->index]->kickfd = -1; > > again unsigned > > > } > > > > return 0; > >diff --git a/lib/librte_vhost/virtio-net.c b/lib/librte_vhost/virtio-net.c > >index deac6b9..643a92e 100644 > >--- a/lib/librte_vhost/virtio-net.c > >+++ b/lib/librte_vhost/virtio-net.c > >@@ -36,6 +36,7 @@ > > #include > > #include > > #include > >+#include > > #include > > #include > > #ifdef RTE_LIBRTE_VHOST_NUMA > >@@ -178,6 +179,15 @@ add_config_ll_entry(struct virtio_net_config_ll *new_ll_dev) > > > > } > > > >+static void > >+cleanup_vq(struct vhost_virtqueue *vq) > >+{ > >+ if (vq->callfd >= 0) > >+ close(vq->callfd); > >+ if (vq->kickfd >= 0) > >+ close(vq->kickfd); > > both always >=0 > > >+} > >+ > > /* > > * Unmap any memory, close any file descriptors and > > * free any memory owned by a device. > >@@ -185,6 +195,8 @@ add_config_ll_entry(struct virtio_net_config_ll *new_ll_dev) > > static void > > cleanup_device(struct virtio_net *dev) > > { > >+ uint32_t i; > >+ > > /* Unmap QEMU memory file if mapped. */ > > if (dev->mem) { > > munmap((void *)(uintptr_t)dev->mem->mapped_address, > >@@ -192,15 +204,10 @@ cleanup_device(struct virtio_net *dev) > > free(dev->mem); > > } > > > >- /* Close any event notifiers opened by device. */ > >- if (dev->virtqueue[VIRTIO_RXQ]->callfd >= 0) > >- close(dev->virtqueue[VIRTIO_RXQ]->callfd); > >- if (dev->virtqueue[VIRTIO_RXQ]->kickfd >= 0) > >- close(dev->virtqueue[VIRTIO_RXQ]->kickfd); > >- if (dev->virtqueue[VIRTIO_TXQ]->callfd >= 0) > >- close(dev->virtqueue[VIRTIO_TXQ]->callfd); > >- if (dev->virtqueue[VIRTIO_TXQ]->kickfd >= 0) > >- close(dev->virtqueue[VIRTIO_TXQ]->kickfd); > >+ for (i = 0; i < dev->virt_qp_nb; i++) { > >+ cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ]); > >+ cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ]); > >+ } > > } > > > > /* > >@@ -209,9 +216,11 @@ cleanup_device(struct virtio_net *dev) > > static void > > free_device(struct virtio_net_config_ll *ll_dev) > > { > >- /* Free any malloc'd memory */ > >- rte_free(ll_dev->dev.virtqueue[VIRTIO_RXQ]); > >- rte_free(ll_dev->dev.virtqueue[VIRTIO_TXQ]); > >+ uint32_t i; > >+ > >+ for (i = 0; i < ll_dev->dev.virt_qp_nb; i++) > >+ rte_free(ll_dev->dev.virtqueue[i * VIRTIO_QNUM]); > >+ > > rte_free(ll_dev); > > } > > > >@@ -244,6 +253,50 @@ rm_config_ll_entry(struct virtio_net_config_ll *ll_dev, > > } > > } > > > >+static void > >+init_vring_queue(struct vhost_virtqueue *vq) > >+{ > >+ memset(vq, 0, sizeof(struct vhost_virtqueue)); > >+ > >+ vq->kickfd = -1; > >+ vq->callfd = -1; > > same here > > >+ > >+ /* Backends are set to -1 indicating an inactive device. */ > >+ vq->backend = -1; > >+} > >+ > >+static void > >+init_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) > >+{ > >+ init_vring_queue(dev->virtqueue[qp_idx * VIRTIO_QNUM + VIRTIO_RXQ]); > >+ init_vring_queue(dev->virtqueue[qp_idx * VIRTIO_QNUM + VIRTIO_TXQ]); > >+} > >+ > >+static int > >+alloc_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) > >+{ > >+ struct vhost_virtqueue *virtqueue = NULL; > >+ uint32_t virt_rx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_RXQ; > >+ uint32_t virt_tx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_TXQ; > >+ > >+ virtqueue = rte_malloc(NULL, > >+ sizeof(struct vhost_virtqueue) * VIRTIO_QNUM, 0); > >+ if (virtqueue == NULL) { > >+ RTE_LOG(ERR, VHOST_CONFIG, > >+ "Failed to allocate memory for virt qp:%d.\n", qp_idx); > >+ return -1; > >+ } > >+ > >+ dev->virtqueue[virt_rx_q_idx] = virtqueue; > >+ dev->virtqueue[virt_tx_q_idx] = virtqueue + VIRTIO_TXQ; > >+ > >+ init_vring_queue_pair(dev, qp_idx); > >+ > >+ dev->virt_qp_nb += 1; > >+ > >+ return 0; > >+} > >+ > > /* > > * Initialise all variables in device structure. > > */ > >@@ -251,6 +304,7 @@ static void > > init_device(struct virtio_net *dev) > > { > > uint64_t vq_offset; > >+ uint32_t i; > > > > /* > > * Virtqueues have already been malloced so > >@@ -261,17 +315,9 @@ init_device(struct virtio_net *dev) > > /* Set everything to 0. */ > > memset((void *)(uintptr_t)((uint64_t)(uintptr_t)dev + vq_offset), 0, > > (sizeof(struct virtio_net) - (size_t)vq_offset)); > >- memset(dev->virtqueue[VIRTIO_RXQ], 0, sizeof(struct vhost_virtqueue)); > >- memset(dev->virtqueue[VIRTIO_TXQ], 0, sizeof(struct vhost_virtqueue)); > > > >- dev->virtqueue[VIRTIO_RXQ]->kickfd = -1; > >- dev->virtqueue[VIRTIO_RXQ]->callfd = -1; > >- dev->virtqueue[VIRTIO_TXQ]->kickfd = -1; > >- dev->virtqueue[VIRTIO_TXQ]->callfd = -1; > >- > >- /* Backends are set to -1 indicating an inactive device. */ > >- dev->virtqueue[VIRTIO_RXQ]->backend = VIRTIO_DEV_STOPPED; > >- dev->virtqueue[VIRTIO_TXQ]->backend = VIRTIO_DEV_STOPPED; > >+ for (i = 0; i < dev->virt_qp_nb; i++) > >+ init_vring_queue_pair(dev, i); > > } > > > > /* > >@@ -283,7 +329,6 @@ static int > > new_device(struct vhost_device_ctx ctx) > > { > > struct virtio_net_config_ll *new_ll_dev; > >- struct vhost_virtqueue *virtqueue_rx, *virtqueue_tx; > > > > /* Setup device and virtqueues. */ > > new_ll_dev = rte_malloc(NULL, sizeof(struct virtio_net_config_ll), 0); > >@@ -294,28 +339,6 @@ new_device(struct vhost_device_ctx ctx) > > return -1; > > } > > > >- virtqueue_rx = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); > >- if (virtqueue_rx == NULL) { > >- rte_free(new_ll_dev); > >- RTE_LOG(ERR, VHOST_CONFIG, > >- "(%"PRIu64") Failed to allocate memory for rxq.\n", > >- ctx.fh); > >- return -1; > >- } > >- > >- virtqueue_tx = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); > >- if (virtqueue_tx == NULL) { > >- rte_free(virtqueue_rx); > >- rte_free(new_ll_dev); > >- RTE_LOG(ERR, VHOST_CONFIG, > >- "(%"PRIu64") Failed to allocate memory for txq.\n", > >- ctx.fh); > >- return -1; > >- } > >- > >- new_ll_dev->dev.virtqueue[VIRTIO_RXQ] = virtqueue_rx; > >- new_ll_dev->dev.virtqueue[VIRTIO_TXQ] = virtqueue_tx; > >- > > /* Initialise device and virtqueues. */ > > init_device(&new_ll_dev->dev); > > > >@@ -680,13 +703,21 @@ set_vring_call(struct vhost_device_ctx ctx, struct vhost_vring_file *file) > > { > > struct virtio_net *dev; > > struct vhost_virtqueue *vq; > >+ uint32_t cur_qp_idx = file->index / VIRTIO_QNUM; > > > > dev = get_device(ctx); > > if (dev == NULL) > > return -1; > > > >+ /* alloc vring queue pair if it is a new queue pair */ > >+ if (cur_qp_idx + 1 > dev->virt_qp_nb) { > >+ if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0) > >+ return -1; > >+ } > >+ > > /* file->index refers to the queue index. The txq is 1, rxq is 0. */ > > vq = dev->virtqueue[file->index]; > >+ assert(vq != NULL); > > > > if (vq->callfd >= 0) > > close(vq->callfd); > > > > > I hope I helped, > Thanks, > Marcel >