From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-wi0-f169.google.com (mail-wi0-f169.google.com [209.85.212.169]) by dpdk.org (Postfix) with ESMTP id 8285E1DB1 for ; Sun, 20 Sep 2015 15:58:44 +0200 (CEST) Received: by wicge5 with SMTP id ge5so84492397wic.0 for ; Sun, 20 Sep 2015 06:58:44 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=reply-to:subject:references:to:cc:from:message-id:date:user-agent :mime-version:in-reply-to:content-type:content-transfer-encoding; bh=CbdrYtZRBU5hCNbK37Vzf+Dd5r01nfHj+vQCd0GnGq0=; b=cp4hrMp76XmYC7uBHlN+AYnp9y39fffTOh3nhT8/5TxBoeCz6NzcKUkpOREDpIslW/ H+wWwiWQrJFw3BxCVt4vjm3sISxQ4+xAKNWbHD5tdIg+OcQZZQ3/2xqVNkWu8y4/7Mj/ d8uhuBndlE2Z90A3Tl9ZcERSoMaujXOFPBkLl6lEfpUL19zNytPXwhmNJ3QMObMM2B+Y UvAaCoL4DKxtCdLodt+rYnKXRxAErPXC5HHjOxdCJMtJrZKH8XodVWWUXUtv3Ci9dw/A kKGGNaNexYS2aHItXJCdmB/i/i40cRfyq3Egia40jK3XWD06HKXfldnDqCXof90YxobJ 4JzQ== X-Received: by 10.194.121.40 with SMTP id lh8mr2851670wjb.115.1442757524285; Sun, 20 Sep 2015 06:58:44 -0700 (PDT) Received: from [192.168.1.115] ([46.120.6.143]) by smtp.googlemail.com with ESMTPSA id gl4sm19135589wjb.29.2015.09.20.06.58.43 (version=TLSv1.2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 20 Sep 2015 06:58:43 -0700 (PDT) References: <1442589061-19225-1-git-send-email-yuanhan.liu@linux.intel.com> <1442589061-19225-4-git-send-email-yuanhan.liu@linux.intel.com> To: Yuanhan Liu , dev@dpdk.org From: Marcel Apfelbaum Message-ID: <55FEBB92.40403@gmail.com> Date: Sun, 20 Sep 2015 16:58:42 +0300 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Thunderbird/38.1.0 MIME-Version: 1.0 In-Reply-To: <1442589061-19225-4-git-send-email-yuanhan.liu@linux.intel.com> Content-Type: text/plain; charset=windows-1252; format=flowed Content-Transfer-Encoding: 7bit Cc: "Michael S. Tsirkin" Subject: Re: [dpdk-dev] [PATCH v5 resend 03/12] vhost: vring queue setup for multiple queue support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list Reply-To: marcel@redhat.com List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 20 Sep 2015 13:58:44 -0000 On 09/18/2015 06:10 PM, Yuanhan Liu wrote: > All queue pairs, including the default (the first) queue pair, > are allocated dynamically, when a vring_call message is received > first time for a specific queue pair. > > This is a refactor work for enabling vhost-user multiple queue; > it should not break anything as it does no functional changes: > we don't support mq set, so there is only one mq at max. > > This patch is based on Changchun's patch. > > Signed-off-by: Yuanhan Liu > --- > lib/librte_vhost/rte_virtio_net.h | 3 +- > lib/librte_vhost/vhost_user/virtio-net-user.c | 44 +++++----- > lib/librte_vhost/virtio-net.c | 121 ++++++++++++++++---------- > 3 files changed, 102 insertions(+), 66 deletions(-) > > diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h > index e3a21e5..5dd6493 100644 > --- a/lib/librte_vhost/rte_virtio_net.h > +++ b/lib/librte_vhost/rte_virtio_net.h > @@ -96,7 +96,7 @@ struct vhost_virtqueue { > * Device structure contains all configuration information relating to the device. > */ > struct virtio_net { > - struct vhost_virtqueue *virtqueue[VIRTIO_QNUM]; /**< Contains all virtqueue information. */ > + struct vhost_virtqueue *virtqueue[VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX]; /**< Contains all virtqueue information. */ > struct virtio_memory *mem; /**< QEMU memory and memory region information. */ > uint64_t features; /**< Negotiated feature set. */ > uint64_t protocol_features; /**< Negotiated protocol feature set. */ > @@ -104,6 +104,7 @@ struct virtio_net { > uint32_t flags; /**< Device flags. Only used to check if device is running on data core. */ > #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ) > char ifname[IF_NAME_SZ]; /**< Name of the tap device or socket path. */ > + uint32_t virt_qp_nb; /**< number of queue pair we have allocated */ > void *priv; /**< private context */ > } __rte_cache_aligned; > > diff --git a/lib/librte_vhost/vhost_user/virtio-net-user.c b/lib/librte_vhost/vhost_user/virtio-net-user.c > index 360254e..e83d279 100644 > --- a/lib/librte_vhost/vhost_user/virtio-net-user.c > +++ b/lib/librte_vhost/vhost_user/virtio-net-user.c > @@ -206,25 +206,33 @@ err_mmap: > } > Hi, > static int > +vq_is_ready(struct vhost_virtqueue *vq) > +{ > + return vq && vq->desc && > + vq->kickfd != -1 && > + vq->callfd != -1; kickfd and callfd are unsigned > +} > + > +static int > virtio_is_ready(struct virtio_net *dev) > { > struct vhost_virtqueue *rvq, *tvq; > + uint32_t i; > > - /* mq support in future.*/ > - rvq = dev->virtqueue[VIRTIO_RXQ]; > - tvq = dev->virtqueue[VIRTIO_TXQ]; > - if (rvq && tvq && rvq->desc && tvq->desc && > - (rvq->kickfd != -1) && > - (rvq->callfd != -1) && > - (tvq->kickfd != -1) && > - (tvq->callfd != -1)) { > - RTE_LOG(INFO, VHOST_CONFIG, > - "virtio is now ready for processing.\n"); > - return 1; > + for (i = 0; i < dev->virt_qp_nb; i++) { > + rvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ]; > + tvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ]; > + > + if (!vq_is_ready(rvq) || !vq_is_ready(tvq)) { > + RTE_LOG(INFO, VHOST_CONFIG, > + "virtio is not ready for processing.\n"); > + return 0; > + } > } > + > RTE_LOG(INFO, VHOST_CONFIG, > - "virtio isn't ready for processing.\n"); > - return 0; > + "virtio is now ready for processing.\n"); > + return 1; > } > > void > @@ -290,13 +298,9 @@ user_get_vring_base(struct vhost_device_ctx ctx, > * sent and only sent in vhost_vring_stop. > * TODO: cleanup the vring, it isn't usable since here. > */ > - if ((dev->virtqueue[VIRTIO_RXQ]->kickfd) >= 0) { > - close(dev->virtqueue[VIRTIO_RXQ]->kickfd); > - dev->virtqueue[VIRTIO_RXQ]->kickfd = -1; > - } > - if ((dev->virtqueue[VIRTIO_TXQ]->kickfd) >= 0) { > - close(dev->virtqueue[VIRTIO_TXQ]->kickfd); > - dev->virtqueue[VIRTIO_TXQ]->kickfd = -1; > + if ((dev->virtqueue[state->index]->kickfd) >= 0) { always >= 0 > + close(dev->virtqueue[state->index]->kickfd); > + dev->virtqueue[state->index]->kickfd = -1; again unsigned > } > > return 0; > diff --git a/lib/librte_vhost/virtio-net.c b/lib/librte_vhost/virtio-net.c > index deac6b9..643a92e 100644 > --- a/lib/librte_vhost/virtio-net.c > +++ b/lib/librte_vhost/virtio-net.c > @@ -36,6 +36,7 @@ > #include > #include > #include > +#include > #include > #include > #ifdef RTE_LIBRTE_VHOST_NUMA > @@ -178,6 +179,15 @@ add_config_ll_entry(struct virtio_net_config_ll *new_ll_dev) > > } > > +static void > +cleanup_vq(struct vhost_virtqueue *vq) > +{ > + if (vq->callfd >= 0) > + close(vq->callfd); > + if (vq->kickfd >= 0) > + close(vq->kickfd); both always >=0 > +} > + > /* > * Unmap any memory, close any file descriptors and > * free any memory owned by a device. > @@ -185,6 +195,8 @@ add_config_ll_entry(struct virtio_net_config_ll *new_ll_dev) > static void > cleanup_device(struct virtio_net *dev) > { > + uint32_t i; > + > /* Unmap QEMU memory file if mapped. */ > if (dev->mem) { > munmap((void *)(uintptr_t)dev->mem->mapped_address, > @@ -192,15 +204,10 @@ cleanup_device(struct virtio_net *dev) > free(dev->mem); > } > > - /* Close any event notifiers opened by device. */ > - if (dev->virtqueue[VIRTIO_RXQ]->callfd >= 0) > - close(dev->virtqueue[VIRTIO_RXQ]->callfd); > - if (dev->virtqueue[VIRTIO_RXQ]->kickfd >= 0) > - close(dev->virtqueue[VIRTIO_RXQ]->kickfd); > - if (dev->virtqueue[VIRTIO_TXQ]->callfd >= 0) > - close(dev->virtqueue[VIRTIO_TXQ]->callfd); > - if (dev->virtqueue[VIRTIO_TXQ]->kickfd >= 0) > - close(dev->virtqueue[VIRTIO_TXQ]->kickfd); > + for (i = 0; i < dev->virt_qp_nb; i++) { > + cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ]); > + cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ]); > + } > } > > /* > @@ -209,9 +216,11 @@ cleanup_device(struct virtio_net *dev) > static void > free_device(struct virtio_net_config_ll *ll_dev) > { > - /* Free any malloc'd memory */ > - rte_free(ll_dev->dev.virtqueue[VIRTIO_RXQ]); > - rte_free(ll_dev->dev.virtqueue[VIRTIO_TXQ]); > + uint32_t i; > + > + for (i = 0; i < ll_dev->dev.virt_qp_nb; i++) > + rte_free(ll_dev->dev.virtqueue[i * VIRTIO_QNUM]); > + > rte_free(ll_dev); > } > > @@ -244,6 +253,50 @@ rm_config_ll_entry(struct virtio_net_config_ll *ll_dev, > } > } > > +static void > +init_vring_queue(struct vhost_virtqueue *vq) > +{ > + memset(vq, 0, sizeof(struct vhost_virtqueue)); > + > + vq->kickfd = -1; > + vq->callfd = -1; same here > + > + /* Backends are set to -1 indicating an inactive device. */ > + vq->backend = -1; > +} > + > +static void > +init_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) > +{ > + init_vring_queue(dev->virtqueue[qp_idx * VIRTIO_QNUM + VIRTIO_RXQ]); > + init_vring_queue(dev->virtqueue[qp_idx * VIRTIO_QNUM + VIRTIO_TXQ]); > +} > + > +static int > +alloc_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) > +{ > + struct vhost_virtqueue *virtqueue = NULL; > + uint32_t virt_rx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_RXQ; > + uint32_t virt_tx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_TXQ; > + > + virtqueue = rte_malloc(NULL, > + sizeof(struct vhost_virtqueue) * VIRTIO_QNUM, 0); > + if (virtqueue == NULL) { > + RTE_LOG(ERR, VHOST_CONFIG, > + "Failed to allocate memory for virt qp:%d.\n", qp_idx); > + return -1; > + } > + > + dev->virtqueue[virt_rx_q_idx] = virtqueue; > + dev->virtqueue[virt_tx_q_idx] = virtqueue + VIRTIO_TXQ; > + > + init_vring_queue_pair(dev, qp_idx); > + > + dev->virt_qp_nb += 1; > + > + return 0; > +} > + > /* > * Initialise all variables in device structure. > */ > @@ -251,6 +304,7 @@ static void > init_device(struct virtio_net *dev) > { > uint64_t vq_offset; > + uint32_t i; > > /* > * Virtqueues have already been malloced so > @@ -261,17 +315,9 @@ init_device(struct virtio_net *dev) > /* Set everything to 0. */ > memset((void *)(uintptr_t)((uint64_t)(uintptr_t)dev + vq_offset), 0, > (sizeof(struct virtio_net) - (size_t)vq_offset)); > - memset(dev->virtqueue[VIRTIO_RXQ], 0, sizeof(struct vhost_virtqueue)); > - memset(dev->virtqueue[VIRTIO_TXQ], 0, sizeof(struct vhost_virtqueue)); > > - dev->virtqueue[VIRTIO_RXQ]->kickfd = -1; > - dev->virtqueue[VIRTIO_RXQ]->callfd = -1; > - dev->virtqueue[VIRTIO_TXQ]->kickfd = -1; > - dev->virtqueue[VIRTIO_TXQ]->callfd = -1; > - > - /* Backends are set to -1 indicating an inactive device. */ > - dev->virtqueue[VIRTIO_RXQ]->backend = VIRTIO_DEV_STOPPED; > - dev->virtqueue[VIRTIO_TXQ]->backend = VIRTIO_DEV_STOPPED; > + for (i = 0; i < dev->virt_qp_nb; i++) > + init_vring_queue_pair(dev, i); > } > > /* > @@ -283,7 +329,6 @@ static int > new_device(struct vhost_device_ctx ctx) > { > struct virtio_net_config_ll *new_ll_dev; > - struct vhost_virtqueue *virtqueue_rx, *virtqueue_tx; > > /* Setup device and virtqueues. */ > new_ll_dev = rte_malloc(NULL, sizeof(struct virtio_net_config_ll), 0); > @@ -294,28 +339,6 @@ new_device(struct vhost_device_ctx ctx) > return -1; > } > > - virtqueue_rx = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); > - if (virtqueue_rx == NULL) { > - rte_free(new_ll_dev); > - RTE_LOG(ERR, VHOST_CONFIG, > - "(%"PRIu64") Failed to allocate memory for rxq.\n", > - ctx.fh); > - return -1; > - } > - > - virtqueue_tx = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); > - if (virtqueue_tx == NULL) { > - rte_free(virtqueue_rx); > - rte_free(new_ll_dev); > - RTE_LOG(ERR, VHOST_CONFIG, > - "(%"PRIu64") Failed to allocate memory for txq.\n", > - ctx.fh); > - return -1; > - } > - > - new_ll_dev->dev.virtqueue[VIRTIO_RXQ] = virtqueue_rx; > - new_ll_dev->dev.virtqueue[VIRTIO_TXQ] = virtqueue_tx; > - > /* Initialise device and virtqueues. */ > init_device(&new_ll_dev->dev); > > @@ -680,13 +703,21 @@ set_vring_call(struct vhost_device_ctx ctx, struct vhost_vring_file *file) > { > struct virtio_net *dev; > struct vhost_virtqueue *vq; > + uint32_t cur_qp_idx = file->index / VIRTIO_QNUM; > > dev = get_device(ctx); > if (dev == NULL) > return -1; > > + /* alloc vring queue pair if it is a new queue pair */ > + if (cur_qp_idx + 1 > dev->virt_qp_nb) { > + if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0) > + return -1; > + } > + > /* file->index refers to the queue index. The txq is 1, rxq is 0. */ > vq = dev->virtqueue[file->index]; > + assert(vq != NULL); > > if (vq->callfd >= 0) > close(vq->callfd); > I hope I helped, Thanks, Marcel