[dpdk-dev] [RFC 2/6] vhost: optimize enqueue path

[Tiwei Bie <tiwei.bie@intel.com>]

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 lib/librte_vhost/virtio_net.c | 185 ++++++++++++++++++++----------------------
 1 file changed, 88 insertions(+), 97 deletions(-)

diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index c0b4dde..0888d2b 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -583,129 +583,120 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 	return pkt_idx;
 }
 
-static inline int __attribute__((always_inline))
-enqueue_pkt(struct virtio_net *dev, struct vring_desc_1_1 *descs,
-	    uint16_t desc_idx, struct rte_mbuf *m)
-{
-	uint32_t desc_avail, desc_offset;
-	uint32_t mbuf_avail, mbuf_offset;
-	uint32_t cpy_len;
-	struct vring_desc_1_1 *desc;
-	uint64_t desc_addr;
-	struct virtio_net_hdr_mrg_rxbuf *hdr;
-
-	desc = &descs[desc_idx];
-	desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
-	/*
-	 * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
-	 * performance issue with some versions of gcc (4.8.4 and 5.3.0) which
-	 * otherwise stores offset on the stack instead of in a register.
-	 */
-	if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr)
-		return -1;
-
-	rte_prefetch0((void *)(uintptr_t)desc_addr);
-
-	hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr;
-	virtio_enqueue_offload(m, &hdr->hdr);
-	vhost_log_write(dev, desc->addr, dev->vhost_hlen);
-	PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);
-
-	desc_offset = dev->vhost_hlen;
-	desc_avail  = desc->len - dev->vhost_hlen;
-
-	mbuf_avail  = rte_pktmbuf_data_len(m);
-	mbuf_offset = 0;
-	while (mbuf_avail != 0 || m->next != NULL) {
-		/* done with current mbuf, fetch next */
-		if (mbuf_avail == 0) {
-			m = m->next;
-
-			mbuf_offset = 0;
-			mbuf_avail  = rte_pktmbuf_data_len(m);
-		}
-
-		/* done with current desc buf, fetch next */
-		if (desc_avail == 0) {
-			if ((desc->flags & VRING_DESC_F_NEXT) == 0) {
-				/* Room in vring buffer is not enough */
-				return -1;
-			}
-
-			rte_panic("Shouldn't reach here\n");
-			/** NOTE: we should not come here with current
-			    virtio-user implementation **/
-			desc_idx = (desc_idx + 1); // & (vq->size - 1);
-			desc = &descs[desc_idx];
-			if (unlikely(!(desc->flags & DESC_HW)))
-				return -1;
-
-			desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
-			if (unlikely(!desc_addr))
-				return -1;
-
-			desc_offset = 0;
-			desc_avail  = desc->len;
-		}
-
-		cpy_len = RTE_MIN(desc_avail, mbuf_avail);
-		rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
-			cpy_len);
-		vhost_log_write(dev, desc->addr + desc_offset, cpy_len);
-		PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
-			     cpy_len, 0);
-
-		mbuf_avail  -= cpy_len;
-		mbuf_offset += cpy_len;
-		desc_avail  -= cpy_len;
-		desc_offset += cpy_len;
-	}
-
-	return 0;
-}
-
 static inline uint32_t __attribute__((always_inline))
 vhost_enqueue_burst_1_1(struct virtio_net *dev, uint16_t queue_id,
 	      struct rte_mbuf **pkts, uint32_t count)
 {
 	struct vhost_virtqueue *vq;
+	struct vring_desc_1_1 *descs;
+	uint16_t head_idx, idx;
+	uint16_t mask;
 	uint16_t i;
-	uint16_t idx;
-	struct vring_desc_1_1 *desc;
-	uint16_t head_idx;
 
 	vq = dev->virtqueue[queue_id];
 	if (unlikely(vq->enabled == 0))
 		return 0;
 
-	head_idx = vq->last_used_idx;
-	desc = vq->desc_1_1;
-	count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST);
+	descs = vq->desc_1_1;
+	mask = vq->size - 1;
+	head_idx = vq->last_used_idx & mask;
 
 	for (i = 0; i < count; i++) {
+		uint32_t desc_avail, desc_offset;
+		uint32_t mbuf_avail, mbuf_offset;
+		uint32_t cpy_len;
+		struct vring_desc_1_1 *desc;
+		uint64_t desc_addr;
+		struct virtio_net_hdr_mrg_rxbuf *hdr;
+		struct rte_mbuf *m = pkts[i];
+
 		/* XXX: there is an assumption that no desc will be chained */
-		idx = vq->last_used_idx & (vq->size - 1);
-		if (!(desc[idx].flags & DESC_HW))
+		idx = vq->last_used_idx & mask;
+		desc = &descs[idx];
+
+		if (!(desc->flags & DESC_HW))
 			break;
 
-		if (enqueue_pkt(dev, desc, idx, pkts[i]) < 0)
+		desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+		/*
+		 * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
+		 * performance issue with some versions of gcc (4.8.4 and 5.3.0) which
+		 * otherwise stores offset on the stack instead of in a register.
+		 */
+		if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr)
 			break;
 
+		hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr;
+		virtio_enqueue_offload(m, &hdr->hdr);
+		vhost_log_write(dev, desc->addr, dev->vhost_hlen);
+		PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);
+
+		desc_offset = dev->vhost_hlen;
+		desc_avail  = desc->len - dev->vhost_hlen;
+
+		mbuf_avail  = rte_pktmbuf_data_len(m);
+		mbuf_offset = 0;
+		while (mbuf_avail != 0 || m->next != NULL) {
+			/* done with current mbuf, fetch next */
+			if (mbuf_avail == 0) {
+				m = m->next;
+
+				mbuf_offset = 0;
+				mbuf_avail  = rte_pktmbuf_data_len(m);
+			}
+
+			/* done with current desc buf, fetch next */
+			if (desc_avail == 0) {
+				if ((desc->flags & VRING_DESC_F_NEXT) == 0) {
+					/* Room in vring buffer is not enough */
+					goto end_of_tx;
+				}
+
+				rte_panic("Shouldn't reach here\n");
+				/** NOTE: we should not come here with current
+				    virtio-user implementation **/
+				idx = (idx + 1); // & (vq->size - 1);
+				desc = &descs[idx];
+				if (unlikely(!(desc->flags & DESC_HW)))
+					goto end_of_tx;
+
+				desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+				if (unlikely(!desc_addr))
+					goto end_of_tx;
+
+				desc_offset = 0;
+				desc_avail  = desc->len;
+			}
+
+			cpy_len = RTE_MIN(desc_avail, mbuf_avail);
+			rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
+				cpy_len);
+			vhost_log_write(dev, desc->addr + desc_offset, cpy_len);
+			PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
+				     cpy_len, 0);
+
+			mbuf_avail  -= cpy_len;
+			mbuf_offset += cpy_len;
+			desc_avail  -= cpy_len;
+			desc_offset += cpy_len;
+		}
+
 		vq->last_used_idx++;
 	}
+
+end_of_tx:
 	count = i;
 
 	if (count) {
 		for (i = 1; i < count; i++) {
-			idx = (head_idx + i) & (vq->size - 1);
-			desc[idx].len = pkts[i]->pkt_len + dev->vhost_hlen;
-			desc[idx].flags &= ~DESC_HW;
+			idx = (head_idx + i) & mask;
+			descs[idx].len = pkts[i]->pkt_len + dev->vhost_hlen;
+			descs[idx].flags &= ~DESC_HW;
 		}
-		desc[head_idx & (vq->size - 1)].len =
-			pkts[0]->pkt_len + dev->vhost_hlen;
+		descs[head_idx].len = pkts[0]->pkt_len + dev->vhost_hlen;
 		rte_smp_wmb();
-		desc[head_idx & (vq->size - 1)].flags &= ~DESC_HW;
+		descs[head_idx].flags &= ~DESC_HW;
 	}
 
 	return count;
-- 
2.7.4