From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mx1.redhat.com (mx3-rdu2.redhat.com [66.187.233.73]) by dpdk.org (Postfix) with ESMTP id 64A871BBA5 for ; Fri, 22 Jun 2018 15:44:00 +0200 (CEST) Received: from smtp.corp.redhat.com (int-mx04.intmail.prod.int.rdu2.redhat.com [10.11.54.4]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 0715F401EF06; Fri, 22 Jun 2018 13:44:00 +0000 (UTC) Received: from localhost.localdomain (ovpn-112-42.ams2.redhat.com [10.36.112.42]) by smtp.corp.redhat.com (Postfix) with ESMTP id 448CE2026D6B; Fri, 22 Jun 2018 13:43:58 +0000 (UTC) From: Maxime Coquelin To: tiwei.bie@intel.com, zhihong.wang@intel.com, jfreimann@redhat.com, dev@dpdk.org Cc: mst@redhat.com, jasowang@redhat.com, wexu@redhat.com, Maxime Coquelin Date: Fri, 22 Jun 2018 15:43:21 +0200 Message-Id: <20180622134327.18973-10-maxime.coquelin@redhat.com> In-Reply-To: <20180622134327.18973-1-maxime.coquelin@redhat.com> References: <20180622134327.18973-1-maxime.coquelin@redhat.com> X-Scanned-By: MIMEDefang 2.78 on 10.11.54.4 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.5]); Fri, 22 Jun 2018 13:44:00 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.5]); Fri, 22 Jun 2018 13:44:00 +0000 (UTC) for IP:'10.11.54.4' DOMAIN:'int-mx04.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'maxime.coquelin@redhat.com' RCPT:'' Subject: [dpdk-dev] [PATCH v5 09/15] vhost: add shadow used ring support for packed rings X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 Jun 2018 13:44:00 -0000 Signed-off-by: Maxime Coquelin --- lib/librte_vhost/vhost.c | 9 ++++-- lib/librte_vhost/vhost.h | 13 ++++++-- lib/librte_vhost/vhost_user.c | 64 +++++++++++++++++++++++++++++---------- lib/librte_vhost/virtio_net.c | 70 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 132 insertions(+), 24 deletions(-) diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c index a85c6646f..7cbf1eded 100644 --- a/lib/librte_vhost/vhost.c +++ b/lib/librte_vhost/vhost.c @@ -94,9 +94,12 @@ cleanup_device(struct virtio_net *dev, int destroy) } void -free_vq(struct vhost_virtqueue *vq) +free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) { - rte_free(vq->shadow_used_ring); + if (vq_is_packed(dev)) + rte_free(vq->shadow_used_packed); + else + rte_free(vq->shadow_used_split); rte_free(vq->batch_copy_elems); rte_mempool_free(vq->iotlb_pool); rte_free(vq); @@ -111,7 +114,7 @@ free_device(struct virtio_net *dev) uint32_t i; for (i = 0; i < dev->nr_vring; i++) - free_vq(dev->virtqueue[i]); + free_vq(dev, dev->virtqueue[i]); rte_free(dev); } diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index 671b4b3bf..62d49f238 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -80,6 +80,12 @@ struct log_cache_entry { unsigned long val; }; +struct vring_used_elem_packed { + uint32_t id; + uint32_t len; + uint32_t count; +}; + /** * Structure contains variables relevant to RX/TX virtqueues. */ @@ -119,7 +125,10 @@ struct vhost_virtqueue { struct zcopy_mbuf *zmbufs; struct zcopy_mbuf_list zmbuf_list; - struct vring_used_elem *shadow_used_ring; + union { + struct vring_used_elem *shadow_used_split; + struct vring_used_elem_packed *shadow_used_packed; + }; uint16_t shadow_used_idx; struct vhost_vring_addr ring_addrs; @@ -553,7 +562,7 @@ void reset_device(struct virtio_net *dev); void vhost_destroy_device(int); void cleanup_vq(struct vhost_virtqueue *vq, int destroy); -void free_vq(struct vhost_virtqueue *vq); +void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq); int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx); diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c index b6097c085..a08d99314 100644 --- a/lib/librte_vhost/vhost_user.c +++ b/lib/librte_vhost/vhost_user.c @@ -244,7 +244,7 @@ vhost_user_set_features(struct virtio_net *dev, uint64_t features) dev->virtqueue[dev->nr_vring] = NULL; cleanup_vq(vq, 1); - free_vq(vq); + free_vq(dev, vq); } } @@ -293,13 +293,26 @@ vhost_user_set_vring_num(struct virtio_net *dev, TAILQ_INIT(&vq->zmbuf_list); } - vq->shadow_used_ring = rte_malloc(NULL, + if (vq_is_packed(dev)) { + vq->shadow_used_packed = rte_malloc(NULL, + vq->size * + sizeof(struct vring_used_elem_packed), + RTE_CACHE_LINE_SIZE); + if (!vq->shadow_used_packed) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to allocate memory for shadow used ring.\n"); + return -1; + } + + } else { + vq->shadow_used_split = rte_malloc(NULL, vq->size * sizeof(struct vring_used_elem), RTE_CACHE_LINE_SIZE); - if (!vq->shadow_used_ring) { - RTE_LOG(ERR, VHOST_CONFIG, - "failed to allocate memory for shadow used ring.\n"); - return -1; + if (!vq->shadow_used_split) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to allocate memory for shadow used ring.\n"); + return -1; + } } vq->batch_copy_elems = rte_malloc(NULL, @@ -326,7 +339,8 @@ numa_realloc(struct virtio_net *dev, int index) struct virtio_net *old_dev; struct vhost_virtqueue *old_vq, *vq; struct zcopy_mbuf *new_zmbuf; - struct vring_used_elem *new_shadow_used_ring; + struct vring_used_elem *new_shadow_used_split; + struct vring_used_elem_packed *new_shadow_used_packed; struct batch_copy_elem *new_batch_copy_elems; int ret; @@ -361,13 +375,26 @@ numa_realloc(struct virtio_net *dev, int index) vq->zmbufs = new_zmbuf; } - new_shadow_used_ring = rte_malloc_socket(NULL, - vq->size * sizeof(struct vring_used_elem), - RTE_CACHE_LINE_SIZE, - newnode); - if (new_shadow_used_ring) { - rte_free(vq->shadow_used_ring); - vq->shadow_used_ring = new_shadow_used_ring; + if (vq_is_packed(dev)) { + new_shadow_used_packed = rte_malloc_socket(NULL, + vq->size * + sizeof(struct vring_used_elem_packed), + RTE_CACHE_LINE_SIZE, + newnode); + if (new_shadow_used_packed) { + rte_free(vq->shadow_used_packed); + vq->shadow_used_packed = new_shadow_used_packed; + } + } else { + new_shadow_used_split = rte_malloc_socket(NULL, + vq->size * + sizeof(struct vring_used_elem), + RTE_CACHE_LINE_SIZE, + newnode); + if (new_shadow_used_split) { + rte_free(vq->shadow_used_split); + vq->shadow_used_split = new_shadow_used_split; + } } new_batch_copy_elems = rte_malloc_socket(NULL, @@ -1062,8 +1089,13 @@ vhost_user_get_vring_base(struct virtio_net *dev, if (dev->dequeue_zero_copy) free_zmbufs(vq); - rte_free(vq->shadow_used_ring); - vq->shadow_used_ring = NULL; + if (vq_is_packed(dev)) { + rte_free(vq->shadow_used_packed); + vq->shadow_used_packed = NULL; + } else { + rte_free(vq->shadow_used_split); + vq->shadow_used_split = NULL; + } rte_free(vq->batch_copy_elems); vq->batch_copy_elems = NULL; diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 35f8cf90a..9571d5cdc 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -20,6 +20,7 @@ #include "iotlb.h" #include "vhost.h" +#include "virtio-packed.h" #define MAX_PKT_BURST 32 @@ -82,7 +83,7 @@ do_flush_shadow_used_ring_split(struct virtio_net *dev, uint16_t to, uint16_t from, uint16_t size) { rte_memcpy(&vq->used->ring[to], - &vq->shadow_used_ring[from], + &vq->shadow_used_split[from], size * sizeof(struct vring_used_elem)); vhost_log_cache_used_vring(dev, vq, offsetof(struct vring_used, ring[to]), @@ -126,8 +127,71 @@ update_shadow_used_ring_split(struct vhost_virtqueue *vq, { uint16_t i = vq->shadow_used_idx++; - vq->shadow_used_ring[i].id = desc_idx; - vq->shadow_used_ring[i].len = len; + vq->shadow_used_split[i].id = desc_idx; + vq->shadow_used_split[i].len = len; +} + +static __rte_always_inline void +flush_shadow_used_ring_packed(struct virtio_net *dev, + struct vhost_virtqueue *vq) +{ + int i; + uint16_t used_idx = vq->last_used_idx; + + /* Split loop in two to save memory barriers */ + for (i = 0; i < vq->shadow_used_idx; i++) { + vq->desc_packed[used_idx].index = vq->shadow_used_packed[i].id; + vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len; + + used_idx += vq->shadow_used_packed[i].count; + } + + rte_smp_wmb(); + + for (i = 0; i < vq->shadow_used_idx; i++) { + uint16_t flags; + + if (vq->shadow_used_packed[i].len) + flags = VRING_DESC_F_WRITE; + else + flags = 0; + + if (vq->used_wrap_counter) { + flags |= VRING_DESC_F_USED; + flags |= VRING_DESC_F_AVAIL; + } else { + flags &= ~VRING_DESC_F_USED; + flags &= ~VRING_DESC_F_AVAIL; + } + + vq->desc_packed[vq->last_used_idx].flags = flags; + + vhost_log_cache_used_vring(dev, vq, + vq->last_used_idx * + sizeof(struct vring_desc_packed), + sizeof(struct vring_desc_packed)); + + vq->last_used_idx += vq->shadow_used_packed[i].count; + if (vq->last_used_idx >= vq->size) { + vq->used_wrap_counter ^= 1; + vq->last_used_idx -= vq->size; + } + } + + rte_smp_wmb(); + vq->shadow_used_idx = 0; + vhost_log_cache_sync(dev, vq); +} + +static __rte_always_inline void +update_shadow_used_ring_packed(struct vhost_virtqueue *vq, + uint16_t desc_idx, uint16_t len, uint16_t count) +{ + uint16_t i = vq->shadow_used_idx++; + + vq->shadow_used_packed[i].id = desc_idx; + vq->shadow_used_packed[i].len = len; + vq->shadow_used_packed[i].count = count; } static inline void -- 2.14.4