From: Zhihong Wang <zhihong.wang@intel.com>
To: dev@dpdk.org
Cc: maxime.coquelin@redhat.com, yuanhan.liu@linux.intel.com,
Zhihong Wang <zhihong.wang@intel.com>
Subject: [dpdk-dev] [PATCH v2 1/6] vhost: rewrite enqueue
Date: Thu, 18 Aug 2016 02:33:06 -0400 [thread overview]
Message-ID: <1471501991-37257-2-git-send-email-zhihong.wang@intel.com> (raw)
In-Reply-To: <1471501991-37257-1-git-send-email-zhihong.wang@intel.com>
This patch implements the vhost logic from scratch into a single function
designed for high performance and better maintainability.
Signed-off-by: Zhihong Wang <zhihong.wang@intel.com>
---
lib/librte_vhost/vhost_rxtx.c | 212 ++++++++++++++++++++++++++++++++++++++++--
1 file changed, 205 insertions(+), 7 deletions(-)
diff --git a/lib/librte_vhost/vhost_rxtx.c b/lib/librte_vhost/vhost_rxtx.c
index 08a73fd..8e6d782 100644
--- a/lib/librte_vhost/vhost_rxtx.c
+++ b/lib/librte_vhost/vhost_rxtx.c
@@ -91,7 +91,7 @@ is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t qp_nb)
return (is_tx ^ (idx & 1)) == 0 && idx < qp_nb * VIRTIO_QNUM;
}
-static void
+static inline void __attribute__((always_inline))
virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr)
{
if (m_buf->ol_flags & PKT_TX_L4_MASK) {
@@ -533,19 +533,217 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
return pkt_idx;
}
+static inline uint32_t __attribute__((always_inline))
+loop_check(struct vhost_virtqueue *vq, uint16_t avail_idx, uint32_t pkt_left)
+{
+ if (pkt_left == 0 || avail_idx == vq->last_used_idx)
+ return 1;
+
+ return 0;
+}
+
+static inline uint32_t __attribute__((always_inline))
+enqueue_packet(struct virtio_net *dev, struct vhost_virtqueue *vq,
+ uint16_t avail_idx, struct rte_mbuf *mbuf,
+ uint32_t is_mrg_rxbuf)
+{
+ struct virtio_net_hdr_mrg_rxbuf *virtio_hdr;
+ struct vring_desc *desc;
+ uint64_t desc_host_write_addr = 0;
+ uint32_t desc_chain_head = 0;
+ uint32_t desc_chain_len = 0;
+ uint32_t desc_current = 0;
+ uint32_t desc_write_offset = 0;
+ uint32_t mbuf_len = 0;
+ uint32_t mbuf_len_left = 0;
+ uint32_t copy_len = 0;
+ uint32_t extra_buffers = 0;
+ uint32_t used_idx_round = 0;
+
+ /* start with the first mbuf of the packet */
+ mbuf_len = rte_pktmbuf_data_len(mbuf);
+ mbuf_len_left = mbuf_len;
+
+ /* get the current desc */
+ desc_current = vq->avail->ring[(vq->last_used_idx) & (vq->size - 1)];
+ desc_chain_head = desc_current;
+ desc = &vq->desc[desc_current];
+ desc_host_write_addr = gpa_to_vva(dev, desc->addr);
+ if (unlikely(!desc_host_write_addr))
+ goto error;
+
+ /* handle virtio header */
+ virtio_hdr = (struct virtio_net_hdr_mrg_rxbuf *)
+ (uintptr_t)desc_host_write_addr;
+ memset((void *)(uintptr_t)&(virtio_hdr->hdr),
+ 0, dev->vhost_hlen);
+ virtio_enqueue_offload(mbuf, &(virtio_hdr->hdr));
+ vhost_log_write(dev, desc->addr, dev->vhost_hlen);
+ desc_write_offset = dev->vhost_hlen;
+ desc_chain_len = desc_write_offset;
+ desc_host_write_addr += desc_write_offset;
+ if (is_mrg_rxbuf)
+ virtio_hdr->num_buffers = 1;
+
+ /* start copy from mbuf to desc */
+ while (1) {
+ /* get the next mbuf if the current done */
+ if (!mbuf_len_left) {
+ if (mbuf->next) {
+ mbuf = mbuf->next;
+ mbuf_len = rte_pktmbuf_data_len(mbuf);
+ mbuf_len_left = mbuf_len;
+ } else
+ break;
+ }
+
+ /* get the next desc if the current done */
+ if (desc->len <= desc_write_offset) {
+ if (desc->flags & VRING_DESC_F_NEXT) {
+ /* go on with the current desc chain */
+ desc_write_offset = 0;
+ desc_current = desc->next;
+ desc = &vq->desc[desc_current];
+ desc_host_write_addr =
+ gpa_to_vva(dev, desc->addr);
+ if (unlikely(!desc_host_write_addr))
+ goto rollback;
+ } else if (is_mrg_rxbuf) {
+ /* start with the next desc chain */
+ used_idx_round = vq->last_used_idx
+ & (vq->size - 1);
+ vq->used->ring[used_idx_round].id =
+ desc_chain_head;
+ vq->used->ring[used_idx_round].len =
+ desc_chain_len;
+ vhost_log_used_vring(dev, vq,
+ offsetof(struct vring_used,
+ ring[used_idx_round]),
+ sizeof(vq->used->ring[
+ used_idx_round]));
+ vq->last_used_idx++;
+ extra_buffers++;
+ virtio_hdr->num_buffers++;
+ if (avail_idx == vq->last_used_idx)
+ goto rollback;
+
+ desc_current =
+ vq->avail->ring[(vq->last_used_idx) &
+ (vq->size - 1)];
+ desc_chain_head = desc_current;
+ desc = &vq->desc[desc_current];
+ desc_host_write_addr =
+ gpa_to_vva(dev, desc->addr);
+ if (unlikely(!desc_host_write_addr))
+ goto rollback;
+
+ desc_chain_len = 0;
+ desc_write_offset = 0;
+ } else
+ goto rollback;
+ }
+
+ /* copy mbuf data */
+ copy_len = RTE_MIN(desc->len - desc_write_offset,
+ mbuf_len_left);
+ rte_memcpy((void *)(uintptr_t)desc_host_write_addr,
+ rte_pktmbuf_mtod_offset(mbuf, void *,
+ mbuf_len - mbuf_len_left),
+ copy_len);
+ vhost_log_write(dev, desc->addr + desc_write_offset,
+ copy_len);
+ mbuf_len_left -= copy_len;
+ desc_write_offset += copy_len;
+ desc_host_write_addr += copy_len;
+ desc_chain_len += copy_len;
+ }
+
+ used_idx_round = vq->last_used_idx & (vq->size - 1);
+ vq->used->ring[used_idx_round].id = desc_chain_head;
+ vq->used->ring[used_idx_round].len = desc_chain_len;
+ vhost_log_used_vring(dev, vq,
+ offsetof(struct vring_used, ring[used_idx_round]),
+ sizeof(vq->used->ring[used_idx_round]));
+ vq->last_used_idx++;
+
+ return 0;
+
+rollback:
+ /* rollback on any error if last_used_idx update on-the-fly */
+ if (is_mrg_rxbuf)
+ vq->last_used_idx -= extra_buffers;
+
+error:
+ return 1;
+}
+
+static inline void __attribute__((always_inline))
+notify_guest(struct virtio_net *dev, struct vhost_virtqueue *vq)
+{
+ rte_smp_wmb();
+ vq->used->idx = vq->last_used_idx;
+ vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx),
+ sizeof(vq->used->idx));
+ rte_mb();
+ if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)
+ && (vq->callfd >= 0))
+ eventfd_write(vq->callfd, (eventfd_t)1);
+}
+
uint16_t
rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
struct rte_mbuf **pkts, uint16_t count)
{
- struct virtio_net *dev = get_device(vid);
+ struct vhost_virtqueue *vq;
+ struct virtio_net *dev;
+ uint32_t pkt_idx = 0;
+ uint32_t pkt_left = 0;
+ uint32_t pkt_sent = 0;
+ uint32_t is_mrg_rxbuf = 0;
+ uint16_t avail_idx = 0;
+
+ /* precheck */
+ if (unlikely(count == 0))
+ return 0;
- if (!dev)
+ count = RTE_MIN((uint32_t)MAX_PKT_BURST, count);
+
+ dev = get_device(vid);
+ if (unlikely(!dev))
return 0;
- if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF))
- return virtio_dev_merge_rx(dev, queue_id, pkts, count);
- else
- return virtio_dev_rx(dev, queue_id, pkts, count);
+ if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->virt_qp_nb)))
+ return 0;
+
+ vq = dev->virtqueue[queue_id];
+ if (unlikely(!vq->enabled))
+ return 0;
+
+ if (dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF))
+ is_mrg_rxbuf = 1;
+
+ /* start enqueuing packets 1 by 1 */
+ pkt_idx = 0;
+ pkt_left = count;
+ avail_idx = *((volatile uint16_t *)&vq->avail->idx);
+ while (1) {
+ if (loop_check(vq, avail_idx, pkt_left))
+ break;
+
+ if (enqueue_packet(dev, vq, avail_idx, pkts[pkt_idx],
+ is_mrg_rxbuf))
+ break;
+
+ pkt_idx++;
+ pkt_sent++;
+ pkt_left--;
+ }
+
+ /* update used idx and kick the guest if necessary */
+ if (pkt_sent)
+ notify_guest(dev, vq);
+
+ return pkt_sent;
}
static void
--
2.7.4
next prev parent reply other threads:[~2016-08-18 13:41 UTC|newest]
Thread overview: 140+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-08-16 3:50 [dpdk-dev] [PATCH] optimize vhost enqueue Zhihong Wang
2016-08-16 13:59 ` Maxime Coquelin
2016-08-17 1:45 ` Wang, Zhihong
2016-08-17 2:38 ` Yuanhan Liu
2016-08-17 6:41 ` Wang, Zhihong
2016-08-17 9:17 ` Maxime Coquelin
2016-08-17 9:51 ` Yuanhan Liu
2016-08-18 13:44 ` Wang, Zhihong
2016-08-17 10:07 ` Wang, Zhihong
2016-08-18 6:33 ` [dpdk-dev] [PATCH v2 0/6] vhost: optimize enqueue Zhihong Wang
2016-08-18 6:33 ` Zhihong Wang [this message]
2016-08-19 2:39 ` [dpdk-dev] [PATCH v2 1/6] vhost: rewrite enqueue Yuanhan Liu
2016-08-19 7:07 ` Wang, Zhihong
2016-08-18 6:33 ` [dpdk-dev] [PATCH v2 2/6] vhost: remove obsolete Zhihong Wang
2016-08-19 2:32 ` Yuanhan Liu
2016-08-19 7:08 ` Wang, Zhihong
2016-08-18 6:33 ` [dpdk-dev] [PATCH v2 3/6] vhost: remove useless volatile Zhihong Wang
2016-08-18 6:33 ` [dpdk-dev] [PATCH v2 4/6] vhost: add desc prefetch Zhihong Wang
2016-08-18 6:33 ` [dpdk-dev] [PATCH v2 5/6] vhost: batch update used ring Zhihong Wang
2016-08-18 6:33 ` [dpdk-dev] [PATCH v2 6/6] vhost: optimize cache access Zhihong Wang
2016-08-19 5:43 ` [dpdk-dev] [PATCH v3 0/5] vhost: optimize enqueue Zhihong Wang
2016-08-19 5:43 ` [dpdk-dev] [PATCH v3 1/5] vhost: rewrite enqueue Zhihong Wang
2016-08-22 9:35 ` Maxime Coquelin
2016-08-23 2:27 ` Wang, Zhihong
2016-08-25 4:00 ` Yuanhan Liu
2016-08-19 5:43 ` [dpdk-dev] [PATCH v3 2/5] vhost: remove useless volatile Zhihong Wang
2016-08-19 5:43 ` [dpdk-dev] [PATCH v3 3/5] vhost: add desc prefetch Zhihong Wang
2016-08-19 5:43 ` [dpdk-dev] [PATCH v3 4/5] vhost: batch update used ring Zhihong Wang
2016-08-25 3:48 ` Yuanhan Liu
2016-08-25 5:19 ` Wang, Zhihong
2016-08-19 5:43 ` [dpdk-dev] [PATCH v3 5/5] vhost: optimize cache access Zhihong Wang
2016-08-22 8:11 ` [dpdk-dev] [PATCH v3 0/5] vhost: optimize enqueue Maxime Coquelin
2016-08-22 10:01 ` Maxime Coquelin
2016-08-22 10:35 ` Thomas Monjalon
2016-08-24 3:37 ` Wang, Zhihong
2016-08-23 2:31 ` Wang, Zhihong
2016-08-23 10:43 ` Wang, Zhihong
2016-08-23 12:16 ` Maxime Coquelin
2016-08-23 12:22 ` Yuanhan Liu
2016-08-23 2:15 ` Wang, Zhihong
2016-09-21 8:50 ` Jianbo Liu
2016-09-21 9:27 ` Wang, Zhihong
2016-09-21 12:54 ` Jianbo Liu
2016-09-22 2:11 ` Wang, Zhihong
2016-09-22 2:29 ` Yuanhan Liu
2016-09-22 5:47 ` Jianbo Liu
2016-09-22 6:58 ` Wang, Zhihong
2016-09-22 9:01 ` Jianbo Liu
2016-09-22 10:04 ` Wang, Zhihong
2016-09-22 14:41 ` Jianbo Liu
2016-09-23 2:56 ` Wang, Zhihong
2016-09-23 10:41 ` Jianbo Liu
2016-09-23 13:41 ` Thomas Monjalon
2016-09-25 5:41 ` Wang, Zhihong
2016-09-26 5:12 ` Jianbo Liu
2016-09-26 5:25 ` Wang, Zhihong
2016-09-26 5:38 ` Jianbo Liu
2016-09-26 6:00 ` Wang, Zhihong
2016-09-26 4:24 ` Jianbo Liu
2016-09-26 5:37 ` Luke Gorrie
2016-09-26 5:40 ` Jianbo Liu
2016-09-27 10:21 ` Yuanhan Liu
2016-09-27 16:45 ` Wang, Zhihong
2016-10-09 12:09 ` Wang, Zhihong
2016-10-10 2:44 ` Yuanhan Liu
2016-10-10 5:31 ` Jianbo Liu
2016-10-10 6:22 ` Wang, Zhihong
2016-10-10 6:57 ` Jianbo Liu
2016-10-10 7:25 ` Wang, Zhihong
2016-10-12 2:53 ` Yuanhan Liu
2016-10-12 12:22 ` Wang, Zhihong
2016-10-12 15:31 ` Thomas Monjalon
2016-10-13 1:21 ` Wang, Zhihong
2016-10-13 3:51 ` Jianbo Liu
2016-10-13 5:33 ` Yuanhan Liu
2016-10-13 5:35 ` Yuanhan Liu
2016-10-13 6:02 ` Wang, Zhihong
2016-10-13 7:54 ` Maxime Coquelin
2016-10-13 9:23 ` Maxime Coquelin
2016-10-14 10:11 ` Yuanhan Liu
2016-08-30 3:35 ` [dpdk-dev] [PATCH v4 0/6] " Zhihong Wang
2016-08-30 3:35 ` [dpdk-dev] [PATCH v4 1/6] vhost: fix windows vm hang Zhihong Wang
2016-09-05 5:24 ` [dpdk-dev] [dpdk-stable] " Yuanhan Liu
2016-09-05 5:25 ` Wang, Zhihong
2016-09-05 5:40 ` Yuanhan Liu
2016-08-30 3:36 ` [dpdk-dev] [PATCH v4 2/6] vhost: rewrite enqueue Zhihong Wang
2016-09-05 6:39 ` Yuanhan Liu
2016-09-07 5:33 ` Yuanhan Liu
2016-09-07 5:39 ` Wang, Zhihong
2016-08-30 3:36 ` [dpdk-dev] [PATCH v4 3/6] vhost: remove useless volatile Zhihong Wang
2016-08-30 3:36 ` [dpdk-dev] [PATCH v4 4/6] vhost: add desc prefetch Zhihong Wang
2016-08-30 3:36 ` [dpdk-dev] [PATCH v4 5/6] vhost: batch update used ring Zhihong Wang
2016-08-30 3:36 ` [dpdk-dev] [PATCH v4 6/6] vhost: optimize cache access Zhihong Wang
2016-09-09 3:39 ` [dpdk-dev] [PATCH v5 0/6] vhost: optimize enqueue Zhihong Wang
2016-09-09 3:39 ` [dpdk-dev] [PATCH v5 1/6] vhost: fix windows vm hang Zhihong Wang
2016-09-09 3:39 ` [dpdk-dev] [PATCH v5 2/6] vhost: rewrite enqueue Zhihong Wang
2016-09-12 15:42 ` Maxime Coquelin
2016-09-14 8:20 ` Wang, Zhihong
2016-09-15 16:35 ` Maxime Coquelin
2016-09-12 16:26 ` Maxime Coquelin
2016-09-14 8:22 ` Wang, Zhihong
2016-09-18 14:19 ` Yuanhan Liu
2016-09-19 3:29 ` Wang, Zhihong
2016-09-09 3:39 ` [dpdk-dev] [PATCH v5 3/6] vhost: remove useless volatile Zhihong Wang
2016-09-09 3:39 ` [dpdk-dev] [PATCH v5 4/6] vhost: add desc prefetch Zhihong Wang
2016-09-09 3:39 ` [dpdk-dev] [PATCH v5 5/6] vhost: batch update used ring Zhihong Wang
2016-09-12 15:45 ` Maxime Coquelin
2016-09-14 8:43 ` Wang, Zhihong
2016-09-15 16:38 ` Maxime Coquelin
2016-09-18 2:55 ` Yuanhan Liu
2016-09-18 2:57 ` Wang, Zhihong
2016-09-09 3:39 ` [dpdk-dev] [PATCH v5 6/6] vhost: optimize cache access Zhihong Wang
2016-09-12 13:52 ` [dpdk-dev] [PATCH v5 0/6] vhost: optimize enqueue Maxime Coquelin
2016-09-12 13:56 ` Maxime Coquelin
2016-09-12 14:01 ` Yuanhan Liu
2016-09-20 2:00 ` [dpdk-dev] [PATCH v6 " Zhihong Wang
2016-09-20 2:00 ` [dpdk-dev] [PATCH v6 1/6] vhost: fix windows vm hang Zhihong Wang
2016-10-13 6:18 ` [dpdk-dev] [dpdk-stable] " Yuanhan Liu
2016-09-20 2:00 ` [dpdk-dev] [PATCH v6 2/6] vhost: rewrite enqueue Zhihong Wang
2016-09-22 9:58 ` Jianbo Liu
2016-09-22 10:13 ` Wang, Zhihong
2016-09-20 2:00 ` [dpdk-dev] [PATCH v6 3/6] vhost: remove useless volatile Zhihong Wang
2016-09-20 2:00 ` [dpdk-dev] [PATCH v6 4/6] vhost: add desc prefetch Zhihong Wang
2016-09-20 2:00 ` [dpdk-dev] [PATCH v6 5/6] vhost: batch update used ring Zhihong Wang
2016-09-20 2:00 ` [dpdk-dev] [PATCH v6 6/6] vhost: optimize cache access Zhihong Wang
2016-09-21 2:26 ` [dpdk-dev] [PATCH v6 0/6] vhost: optimize enqueue Yuanhan Liu
2016-09-21 4:39 ` Maxime Coquelin
2016-10-14 9:34 ` [dpdk-dev] [PATCH v7 0/7] vhost: optimize mergeable Rx path Yuanhan Liu
2016-10-14 9:34 ` [dpdk-dev] [PATCH v7 1/7] vhost: remove useless volatile Yuanhan Liu
2016-10-14 9:34 ` [dpdk-dev] [PATCH v7 2/7] vhost: optimize cache access Yuanhan Liu
2016-10-14 9:34 ` [dpdk-dev] [PATCH v7 3/7] vhost: simplify mergeable Rx vring reservation Yuanhan Liu
2016-10-25 22:08 ` Thomas Monjalon
2016-10-26 2:56 ` Yuanhan Liu
2016-10-14 9:34 ` [dpdk-dev] [PATCH v7 4/7] vhost: use last avail idx for avail ring reservation Yuanhan Liu
2016-10-14 9:34 ` [dpdk-dev] [PATCH v7 5/7] vhost: shadow used ring update Yuanhan Liu
2016-10-14 9:34 ` [dpdk-dev] [PATCH v7 6/7] vhost: prefetch avail ring Yuanhan Liu
2016-10-14 9:34 ` [dpdk-dev] [PATCH v7 7/7] vhost: retrieve avail head once Yuanhan Liu
2016-10-18 2:25 ` [dpdk-dev] [PATCH v7 0/7] vhost: optimize mergeable Rx path Jianbo Liu
2016-10-18 14:53 ` Maxime Coquelin
2016-10-21 7:51 ` Yuanhan Liu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1471501991-37257-2-git-send-email-zhihong.wang@intel.com \
--to=zhihong.wang@intel.com \
--cc=dev@dpdk.org \
--cc=maxime.coquelin@redhat.com \
--cc=yuanhan.liu@linux.intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).