[dpdk-dev] [RFC 0/6] virtio1.1 prototype updates

[dpdk-dev] [RFC 0/6] virtio1.1 prototype updates

[Tiwei Bie]

This patchset depends on below patchset:

http://dpdk.org/ml/archives/dev/2017-June/068315.html

I'll also collect these public patches into my github repo:

https://github.com/btw616/dpdk-virtio1.1

Best regards,
Tiwei Bie

Tiwei Bie (6):
  net/virtio: optimize the rx path
  vhost: optimize enqueue path
  net/virtio: optimize the tx path
  net/virtio: revert the changes in 18dc1b1ac
  vhost: minor refinement
  virtio1.1: introduce the DESC_WB flag

 drivers/net/virtio/virtio-1.1.h      |   1 +
 drivers/net/virtio/virtio_ethdev.c   |   5 +-
 drivers/net/virtio/virtio_ethdev.h   |   2 +
 drivers/net/virtio/virtio_rxtx.c     | 278 +++++++++++++++++++----------------
 drivers/net/virtio/virtio_rxtx_1.1.c | 112 +++++++-------
 drivers/net/virtio/virtqueue.h       |   2 +
 lib/librte_vhost/virtio-1.1.h        |   1 +
 lib/librte_vhost/virtio_net.c        | 221 ++++++++++++++--------------
 8 files changed, 336 insertions(+), 286 deletions(-)

-- 
2.7.4

[dpdk-dev] [RFC 1/6] net/virtio: optimize the rx path

[Tiwei Bie]

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 drivers/net/virtio/virtio_ethdev.c |   5 +-
 drivers/net/virtio/virtio_ethdev.h |   2 +
 drivers/net/virtio/virtio_rxtx.c   | 255 ++++++++++++++++++++-----------------
 3 files changed, 142 insertions(+), 120 deletions(-)

diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
index 334c4b8..566eede 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1241,7 +1241,10 @@ static void
 rx_func_get(struct rte_eth_dev *eth_dev)
 {
 	struct virtio_hw *hw = eth_dev->data->dev_private;
-	if (0 && vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF))
+
+	if (vtpci_version_1_1(hw))
+		eth_dev->rx_pkt_burst = &virtio_recv_pkts_1_1;
+	else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF))
 		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
 	else
 		eth_dev->rx_pkt_burst = &virtio_recv_pkts;
diff --git a/drivers/net/virtio/virtio_ethdev.h b/drivers/net/virtio/virtio_ethdev.h
index deed34e..8bd699d 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -99,6 +99,8 @@ int  virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 
 uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts);
+uint16_t virtio_recv_pkts_1_1(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts);
 
 uint16_t virtio_recv_mergeable_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 3dc5eaf..3983626 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -116,7 +116,7 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
 }
 
 static inline uint16_t
-virtqueue_dequeue_burst_rx_1_0(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
+virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 			   uint32_t *len, uint16_t num)
 {
 	struct vring_used_elem *uep;
@@ -151,53 +151,6 @@ virtqueue_dequeue_burst_rx_1_0(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 	return i;
 }
 
-static inline uint16_t
-virtqueue_dequeue_burst_rx_1_1(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
-			   uint32_t *len, uint16_t num)
-{
-	struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
-	struct rte_mbuf *cookie;
-	uint16_t used_idx;
-	uint16_t i;
-
-	for (i = 0; i < num ; i++) {
-		used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
-		if ((desc[used_idx].flags & DESC_HW))
-			break;
-
-		len[i] = desc[used_idx].len;
-		cookie = vq->vq_descx[used_idx].cookie;
-
-		if (unlikely(cookie == NULL)) {
-			PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
-				vq->vq_used_cons_idx);
-			break;
-		}
-		vq->vq_descx[used_idx].cookie = NULL;
-
-		rte_prefetch0(cookie);
-#if 0
-		rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
-#endif
-		rx_pkts[i]  = cookie;
-
-		vq->vq_used_cons_idx++;
-		vq->vq_free_cnt++;
-	}
-
-	return i;
-}
-
-static inline uint16_t
-virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
-			   uint32_t *len, uint16_t num)
-{
-	if (vtpci_version_1_1(vq->hw))
-		return virtqueue_dequeue_burst_rx_1_1(vq, rx_pkts, len, num);
-	else
-		return virtqueue_dequeue_burst_rx_1_0(vq, rx_pkts, len, num);
-}
-
 #ifndef DEFAULT_TX_FREE_THRESH
 #define DEFAULT_TX_FREE_THRESH 32
 #endif
@@ -226,9 +179,8 @@ virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
 	}
 }
 
-
 static inline int
-virtqueue_enqueue_recv_refill_1_0(struct virtqueue *vq, struct rte_mbuf *cookie)
+virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
 {
 	struct vq_desc_extra *dxp;
 	struct virtio_hw *hw = vq->hw;
@@ -267,50 +219,6 @@ virtqueue_enqueue_recv_refill_1_0(struct virtqueue *vq, struct rte_mbuf *cookie)
 	return 0;
 }
 
-static inline int
-virtqueue_enqueue_recv_refill_1_1(struct virtqueue *vq, struct rte_mbuf *cookie)
-{
-	struct vq_desc_extra *dxp;
-	struct virtio_hw *hw = vq->hw;
-	uint16_t needed = 1;
-	uint16_t idx;
-	struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
-
-	if (unlikely(vq->vq_free_cnt == 0))
-		return -ENOSPC;
-	if (unlikely(vq->vq_free_cnt < needed))
-		return -EMSGSIZE;
-
-	idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1);
-	if (unlikely(desc[idx].flags & DESC_HW))
-		return -EFAULT;
-
-	dxp = &vq->vq_descx[idx];
-	dxp->cookie = cookie;
-	dxp->ndescs = needed;
-
-	desc[idx].addr =
-		VIRTIO_MBUF_ADDR(cookie, vq) +
-		RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
-	desc[idx].len =
-		cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
-	desc[idx].flags =  VRING_DESC_F_WRITE;
-	vq->vq_desc_head_idx++;
-
-	vq->vq_free_cnt -= needed;
-
-	return 0;
-}
-
-static inline int
-virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
-{
-	if (vtpci_version_1_1(vq->hw))
-		return virtqueue_enqueue_recv_refill_1_1(vq, cookie);
-	else
-		return virtqueue_enqueue_recv_refill_1_0(vq, cookie);
-}
-
 static inline void
 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 		       uint16_t needed)
@@ -395,6 +303,30 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 	dev->data->rx_queues[queue_idx] = rxvq;
 
+	if (vtpci_version_1_1(hw)) {
+		struct vring_desc_1_1 *desc;
+		struct vq_desc_extra *dxp;
+
+		for (desc_idx = 0; desc_idx < vq->vq_nentries;
+				desc_idx++) {
+			m = rte_mbuf_raw_alloc(rxvq->mpool);
+			if (unlikely(m == NULL))
+				return -ENOMEM;
+
+			dxp = &vq->vq_descx[desc_idx];
+			dxp->cookie = m;
+			dxp->ndescs = 1;
+
+			desc = &vq->vq_ring.desc_1_1[desc_idx];
+			desc->addr = VIRTIO_MBUF_ADDR(m, vq) +
+				RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
+			desc->len = m->buf_len - RTE_PKTMBUF_HEADROOM +
+				hw->vtnet_hdr_size;
+			desc->flags = VRING_DESC_F_WRITE | DESC_HW;
+		}
+
+		return 0;
+	}
 
 	/* Allocate blank mbufs for the each rx descriptor */
 	nbufs = 0;
@@ -692,6 +624,114 @@ rx_offload_enabled(struct virtio_hw *hw)
 		vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
 }
 
+uint16_t
+virtio_recv_pkts_1_1(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	struct virtnet_rx *rxvq = rx_queue;
+	struct virtqueue *vq = rxvq->vq;
+	struct virtio_hw *hw = vq->hw;
+	struct rte_mbuf *rxm, *nmb;
+	uint16_t nb_rx;
+	uint32_t len;
+	uint32_t i;
+	uint32_t hdr_size;
+	int offload;
+	struct virtio_net_hdr *hdr;
+	struct vring_desc_1_1 *descs = vq->vq_ring.desc_1_1;
+	struct vring_desc_1_1 *desc;
+	uint16_t used_idx = vq->vq_used_cons_idx;
+	struct vq_desc_extra *dxp;
+
+	nb_rx = 0;
+	if (unlikely(hw->started == 0))
+		return nb_rx;
+
+	hdr_size = hw->vtnet_hdr_size;
+	offload = rx_offload_enabled(hw);
+
+	for (i = 0; i < nb_pkts; i++) {
+		desc = &descs[used_idx & (vq->vq_nentries - 1)];
+		if (desc->flags & DESC_HW)
+			break;
+
+		nmb = rte_mbuf_raw_alloc(rxvq->mpool);
+		if (unlikely(nmb == NULL)) {
+			struct rte_eth_dev *dev
+				= &rte_eth_devices[rxvq->port_id];
+			dev->data->rx_mbuf_alloc_failed++;
+			break;
+		}
+
+		dxp = &vq->vq_descx[used_idx & (vq->vq_nentries - 1)];
+
+		len = desc->len;
+		rxm = dxp->cookie;
+		dxp->cookie = nmb;
+		dxp->ndescs = 1;
+
+		desc->addr = VIRTIO_MBUF_ADDR(nmb, vq) +
+			RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
+		desc->len = nmb->buf_len - RTE_PKTMBUF_HEADROOM +
+			hw->vtnet_hdr_size;
+		desc->flags = VRING_DESC_F_WRITE;
+
+		PMD_RX_LOG(DEBUG, "packet len:%d", len);
+
+		if (unlikely(len < hdr_size + ETHER_HDR_LEN)) {
+			PMD_RX_LOG(ERR, "Packet drop");
+			rte_pktmbuf_free(rxm);
+			rxvq->stats.errors++;
+			continue;
+		}
+
+		rxm->port = rxvq->port_id;
+		rxm->data_off = RTE_PKTMBUF_HEADROOM;
+		rxm->ol_flags = 0;
+		rxm->vlan_tci = 0;
+
+		rxm->pkt_len = (uint32_t)(len - hdr_size);
+		rxm->data_len = (uint16_t)(len - hdr_size);
+
+		hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
+			RTE_PKTMBUF_HEADROOM - hdr_size);
+
+		if (hw->vlan_strip)
+			rte_vlan_strip(rxm);
+
+		if (offload && virtio_rx_offload(rxm, hdr) < 0) {
+			rte_pktmbuf_free(rxm);
+			rxvq->stats.errors++;
+			continue;
+		}
+
+		VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
+
+		rxvq->stats.bytes += rxm->pkt_len;
+		virtio_update_packet_stats(&rxvq->stats, rxm);
+
+		used_idx++;
+
+		rx_pkts[nb_rx++] = rxm;
+	}
+
+	if (nb_rx > 0) {
+		uint16_t idx = (vq->vq_used_cons_idx + 1) & (vq->vq_nentries - 1);
+		for (i = 1; i < nb_rx; i++) {
+			descs[idx].flags |= DESC_HW;
+			idx = (idx + 1) & (vq->vq_nentries - 1);
+		}
+
+		rte_smp_wmb();
+		descs[vq->vq_used_cons_idx & (vq->vq_nentries - 1)].flags |= DESC_HW;
+	}
+
+	rxvq->stats.packets += nb_rx;
+
+	vq->vq_used_cons_idx = used_idx;
+
+	return nb_rx;
+}
+
 #define VIRTIO_MBUF_BURST_SZ 64
 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 uint16_t
@@ -709,22 +749,12 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	uint32_t hdr_size;
 	int offload;
 	struct virtio_net_hdr *hdr;
-	uint16_t head_idx, idx;
 
 	nb_rx = 0;
 	if (unlikely(hw->started == 0))
 		return nb_rx;
 
-	/*
-	 * we have no idea to know how many used entries without scanning
-	 * the desc for virtio 1.1. Thus, let's simply set nb_used to nb_pkts
-	 * and let virtqueue_dequeue_burst_rx() to figure out the real
-	 * number.
-	 */
-	if (vtpci_version_1_1(hw))
-		nb_used = nb_pkts;
-	else
-		nb_used = VIRTQUEUE_NUSED(vq);
+	nb_used = VIRTQUEUE_NUSED(vq);
 
 	virtio_rmb();
 
@@ -783,8 +813,6 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	rxvq->stats.packets += nb_rx;
 
-	head_idx = vq->vq_desc_head_idx;
-
 	/* Allocate new mbuf for the used descriptor */
 	error = ENOSPC;
 	int count = 0;
@@ -805,18 +833,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		count++;
 	}
 
-	if (vtpci_version_1_1(hw)) {
-		struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
-		if (count > 0) {
-			rte_smp_wmb();
-			idx = head_idx + 1;
-			while (--count) {
-				desc[idx & (vq->vq_nentries - 1)].flags |= DESC_HW;
-				idx++;
-			}
-			desc[head_idx & (vq->vq_nentries - 1)].flags |= DESC_HW;
-		}
-	} else if (likely(nb_enqueued)) {
+	if (likely(nb_enqueued)) {
 		vq_update_avail_idx(vq);
 
 		if (unlikely(virtqueue_kick_prepare(vq))) {
-- 
2.7.4

[dpdk-dev] [RFC 2/6] vhost: optimize enqueue path

[Tiwei Bie]

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 lib/librte_vhost/virtio_net.c | 185 ++++++++++++++++++++----------------------
 1 file changed, 88 insertions(+), 97 deletions(-)

diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index c0b4dde..0888d2b 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -583,129 +583,120 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
 	return pkt_idx;
 }
 
-static inline int __attribute__((always_inline))
-enqueue_pkt(struct virtio_net *dev, struct vring_desc_1_1 *descs,
-	    uint16_t desc_idx, struct rte_mbuf *m)
-{
-	uint32_t desc_avail, desc_offset;
-	uint32_t mbuf_avail, mbuf_offset;
-	uint32_t cpy_len;
-	struct vring_desc_1_1 *desc;
-	uint64_t desc_addr;
-	struct virtio_net_hdr_mrg_rxbuf *hdr;
-
-	desc = &descs[desc_idx];
-	desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
-	/*
-	 * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
-	 * performance issue with some versions of gcc (4.8.4 and 5.3.0) which
-	 * otherwise stores offset on the stack instead of in a register.
-	 */
-	if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr)
-		return -1;
-
-	rte_prefetch0((void *)(uintptr_t)desc_addr);
-
-	hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr;
-	virtio_enqueue_offload(m, &hdr->hdr);
-	vhost_log_write(dev, desc->addr, dev->vhost_hlen);
-	PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);
-
-	desc_offset = dev->vhost_hlen;
-	desc_avail  = desc->len - dev->vhost_hlen;
-
-	mbuf_avail  = rte_pktmbuf_data_len(m);
-	mbuf_offset = 0;
-	while (mbuf_avail != 0 || m->next != NULL) {
-		/* done with current mbuf, fetch next */
-		if (mbuf_avail == 0) {
-			m = m->next;
-
-			mbuf_offset = 0;
-			mbuf_avail  = rte_pktmbuf_data_len(m);
-		}
-
-		/* done with current desc buf, fetch next */
-		if (desc_avail == 0) {
-			if ((desc->flags & VRING_DESC_F_NEXT) == 0) {
-				/* Room in vring buffer is not enough */
-				return -1;
-			}
-
-			rte_panic("Shouldn't reach here\n");
-			/** NOTE: we should not come here with current
-			    virtio-user implementation **/
-			desc_idx = (desc_idx + 1); // & (vq->size - 1);
-			desc = &descs[desc_idx];
-			if (unlikely(!(desc->flags & DESC_HW)))
-				return -1;
-
-			desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
-			if (unlikely(!desc_addr))
-				return -1;
-
-			desc_offset = 0;
-			desc_avail  = desc->len;
-		}
-
-		cpy_len = RTE_MIN(desc_avail, mbuf_avail);
-		rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
-			cpy_len);
-		vhost_log_write(dev, desc->addr + desc_offset, cpy_len);
-		PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
-			     cpy_len, 0);
-
-		mbuf_avail  -= cpy_len;
-		mbuf_offset += cpy_len;
-		desc_avail  -= cpy_len;
-		desc_offset += cpy_len;
-	}
-
-	return 0;
-}
-
 static inline uint32_t __attribute__((always_inline))
 vhost_enqueue_burst_1_1(struct virtio_net *dev, uint16_t queue_id,
 	      struct rte_mbuf **pkts, uint32_t count)
 {
 	struct vhost_virtqueue *vq;
+	struct vring_desc_1_1 *descs;
+	uint16_t head_idx, idx;
+	uint16_t mask;
 	uint16_t i;
-	uint16_t idx;
-	struct vring_desc_1_1 *desc;
-	uint16_t head_idx;
 
 	vq = dev->virtqueue[queue_id];
 	if (unlikely(vq->enabled == 0))
 		return 0;
 
-	head_idx = vq->last_used_idx;
-	desc = vq->desc_1_1;
-	count = RTE_MIN(count, (uint32_t)MAX_PKT_BURST);
+	descs = vq->desc_1_1;
+	mask = vq->size - 1;
+	head_idx = vq->last_used_idx & mask;
 
 	for (i = 0; i < count; i++) {
+		uint32_t desc_avail, desc_offset;
+		uint32_t mbuf_avail, mbuf_offset;
+		uint32_t cpy_len;
+		struct vring_desc_1_1 *desc;
+		uint64_t desc_addr;
+		struct virtio_net_hdr_mrg_rxbuf *hdr;
+		struct rte_mbuf *m = pkts[i];
+
 		/* XXX: there is an assumption that no desc will be chained */
-		idx = vq->last_used_idx & (vq->size - 1);
-		if (!(desc[idx].flags & DESC_HW))
+		idx = vq->last_used_idx & mask;
+		desc = &descs[idx];
+
+		if (!(desc->flags & DESC_HW))
 			break;
 
-		if (enqueue_pkt(dev, desc, idx, pkts[i]) < 0)
+		desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+		/*
+		 * Checking of 'desc_addr' placed outside of 'unlikely' macro to avoid
+		 * performance issue with some versions of gcc (4.8.4 and 5.3.0) which
+		 * otherwise stores offset on the stack instead of in a register.
+		 */
+		if (unlikely(desc->len < dev->vhost_hlen) || !desc_addr)
 			break;
 
+		hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)desc_addr;
+		virtio_enqueue_offload(m, &hdr->hdr);
+		vhost_log_write(dev, desc->addr, dev->vhost_hlen);
+		PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0);
+
+		desc_offset = dev->vhost_hlen;
+		desc_avail  = desc->len - dev->vhost_hlen;
+
+		mbuf_avail  = rte_pktmbuf_data_len(m);
+		mbuf_offset = 0;
+		while (mbuf_avail != 0 || m->next != NULL) {
+			/* done with current mbuf, fetch next */
+			if (mbuf_avail == 0) {
+				m = m->next;
+
+				mbuf_offset = 0;
+				mbuf_avail  = rte_pktmbuf_data_len(m);
+			}
+
+			/* done with current desc buf, fetch next */
+			if (desc_avail == 0) {
+				if ((desc->flags & VRING_DESC_F_NEXT) == 0) {
+					/* Room in vring buffer is not enough */
+					goto end_of_tx;
+				}
+
+				rte_panic("Shouldn't reach here\n");
+				/** NOTE: we should not come here with current
+				    virtio-user implementation **/
+				idx = (idx + 1); // & (vq->size - 1);
+				desc = &descs[idx];
+				if (unlikely(!(desc->flags & DESC_HW)))
+					goto end_of_tx;
+
+				desc_addr = rte_vhost_gpa_to_vva(dev->mem, desc->addr);
+				if (unlikely(!desc_addr))
+					goto end_of_tx;
+
+				desc_offset = 0;
+				desc_avail  = desc->len;
+			}
+
+			cpy_len = RTE_MIN(desc_avail, mbuf_avail);
+			rte_memcpy((void *)((uintptr_t)(desc_addr + desc_offset)),
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
+				cpy_len);
+			vhost_log_write(dev, desc->addr + desc_offset, cpy_len);
+			PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset),
+				     cpy_len, 0);
+
+			mbuf_avail  -= cpy_len;
+			mbuf_offset += cpy_len;
+			desc_avail  -= cpy_len;
+			desc_offset += cpy_len;
+		}
+
 		vq->last_used_idx++;
 	}
+
+end_of_tx:
 	count = i;
 
 	if (count) {
 		for (i = 1; i < count; i++) {
-			idx = (head_idx + i) & (vq->size - 1);
-			desc[idx].len = pkts[i]->pkt_len + dev->vhost_hlen;
-			desc[idx].flags &= ~DESC_HW;
+			idx = (head_idx + i) & mask;
+			descs[idx].len = pkts[i]->pkt_len + dev->vhost_hlen;
+			descs[idx].flags &= ~DESC_HW;
 		}
-		desc[head_idx & (vq->size - 1)].len =
-			pkts[0]->pkt_len + dev->vhost_hlen;
+		descs[head_idx].len = pkts[0]->pkt_len + dev->vhost_hlen;
 		rte_smp_wmb();
-		desc[head_idx & (vq->size - 1)].flags &= ~DESC_HW;
+		descs[head_idx].flags &= ~DESC_HW;
 	}
 
 	return count;
-- 
2.7.4

[dpdk-dev] [RFC 3/6] net/virtio: optimize the tx path

[Tiwei Bie]

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 drivers/net/virtio/virtio_rxtx_1.1.c | 65 ++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 37 deletions(-)

diff --git a/drivers/net/virtio/virtio_rxtx_1.1.c b/drivers/net/virtio/virtio_rxtx_1.1.c
index 4602e6d..883a027 100644
--- a/drivers/net/virtio/virtio_rxtx_1.1.c
+++ b/drivers/net/virtio/virtio_rxtx_1.1.c
@@ -80,42 +80,6 @@ virtio_xmit_cleanup(struct virtqueue *vq)
 	}
 }
 
-static inline void
-virtio_xmit(struct virtnet_tx *txvq, struct rte_mbuf *mbuf, int first_mbuf)
-{
-	struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
-	struct virtqueue *vq = txvq->vq;
-	struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
-	uint16_t idx;
-	uint16_t head_idx = (vq->vq_avail_idx++) & (vq->vq_nentries - 1);
-	struct vq_desc_extra *dxp;
-
-	idx = head_idx;
-	vq->vq_free_cnt -= mbuf->nb_segs + 1;
-
-	dxp = &vq->vq_descx[idx];
-	if (dxp->cookie != NULL)
-		rte_pktmbuf_free(dxp->cookie);
-	dxp->cookie = mbuf;
-
-	desc[idx].addr  = txvq->virtio_net_hdr_mem +
-			  RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
-	desc[idx].len   = vq->hw->vtnet_hdr_size;
-	desc[idx].flags = VRING_DESC_F_NEXT;
-	if (!first_mbuf)
-		desc[idx].flags |= DESC_HW;
-
-	do {
-		idx = (vq->vq_avail_idx++) & (vq->vq_nentries - 1);
-		desc[idx].addr  = VIRTIO_MBUF_DATA_DMA_ADDR(mbuf, vq);
-		desc[idx].len   = mbuf->data_len;
-		desc[idx].flags = DESC_HW | VRING_DESC_F_NEXT;
-	} while ((mbuf = mbuf->next) != NULL);
-
-	desc[idx].flags &= ~VRING_DESC_F_NEXT;
-
-}
-
 uint16_t
 virtio_xmit_pkts_1_1(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 {
@@ -123,6 +87,9 @@ virtio_xmit_pkts_1_1(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts
 	struct virtqueue *vq = txvq->vq;
 	uint16_t i;
 	uint16_t head_idx = vq->vq_avail_idx;
+	struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
+	uint16_t idx;
+	struct vq_desc_extra *dxp;
 
 	if (unlikely(nb_pkts < 1))
 		return nb_pkts;
@@ -134,6 +101,7 @@ virtio_xmit_pkts_1_1(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts
 
 	for (i = 0; i < nb_pkts; i++) {
 		struct rte_mbuf *txm = tx_pkts[i];
+		struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
 
 		if (unlikely(txm->nb_segs + 1 > vq->vq_free_cnt)) {
 			virtio_xmit_cleanup(vq);
@@ -145,8 +113,31 @@ virtio_xmit_pkts_1_1(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts
 			}
 		}
 
-		virtio_xmit(txvq, txm, i == 0);
 		txvq->stats.bytes += txm->pkt_len;
+
+		vq->vq_free_cnt -= txm->nb_segs + 1;
+
+		idx = (vq->vq_avail_idx++) & (vq->vq_nentries - 1);
+		dxp = &vq->vq_descx[idx];
+		if (dxp->cookie != NULL)
+			rte_pktmbuf_free(dxp->cookie);
+		dxp->cookie = txm;
+
+		desc[idx].addr  = txvq->virtio_net_hdr_mem +
+				  RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
+		desc[idx].len   = vq->hw->vtnet_hdr_size;
+		desc[idx].flags = VRING_DESC_F_NEXT;
+		if (i != 0)
+			desc[idx].flags |= DESC_HW;
+
+		do {
+			idx = (vq->vq_avail_idx++) & (vq->vq_nentries - 1);
+			desc[idx].addr  = VIRTIO_MBUF_DATA_DMA_ADDR(txm, vq);
+			desc[idx].len   = txm->data_len;
+			desc[idx].flags = DESC_HW | VRING_DESC_F_NEXT;
+		} while ((txm = txm->next) != NULL);
+
+		desc[idx].flags &= ~VRING_DESC_F_NEXT;
 	}
 
 	if (likely(i)) {
-- 
2.7.4

[dpdk-dev] [RFC 4/6] net/virtio: revert the changes in 18dc1b1ac

[Tiwei Bie]

Revert the changes in 18dc1b1ac, because we want to test the
performance similar to the real world case.

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 drivers/net/virtio/virtio_rxtx.c | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 3983626..7be9aae 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -139,9 +139,7 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 		}
 
 		rte_prefetch0(cookie);
-#if 0
 		rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
-#endif
 		rx_pkts[i]  = cookie;
 		vq->vq_used_cons_idx++;
 		vq_ring_free_chain(vq, desc_idx);
@@ -512,9 +510,7 @@ static void
 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
 {
 	uint32_t s = mbuf->pkt_len;
-#if 0
 	struct ether_addr *ea;
-#endif
 
 	if (s == 64) {
 		stats->size_bins[1]++;
@@ -533,7 +529,6 @@ virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
 			stats->size_bins[7]++;
 	}
 
-#if 0
 	ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *);
 	if (is_multicast_ether_addr(ea)) {
 		if (is_broadcast_ether_addr(ea))
@@ -541,7 +536,6 @@ virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
 		else
 			stats->multicast++;
 	}
-#endif
 }
 
 /* Optionally fill offload information in structure */
@@ -664,6 +658,9 @@ virtio_recv_pkts_1_1(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts
 
 		dxp = &vq->vq_descx[used_idx & (vq->vq_nentries - 1)];
 
+		rte_packet_prefetch(rte_pktmbuf_mtod(
+				(struct rte_mbuf *)dxp->cookie, void *));
+
 		len = desc->len;
 		rxm = dxp->cookie;
 		dxp->cookie = nmb;
-- 
2.7.4

[dpdk-dev] [RFC 5/6] vhost: minor refinement

[Tiwei Bie]

Manually prefetch the first desc in dequeue path.

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 lib/librte_vhost/virtio_net.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 0888d2b..08d53d9 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -1259,6 +1259,7 @@ vhost_dequeue_burst_1_1(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
 	vq->last_used_idx = desc_idx;
 	if (likely(i)) {
+		rte_prefetch0(&desc[head_idx & (vq->size - 1)]);
 		for (desc_idx = head_idx + 1;
 		     desc_idx != vq->last_used_idx;
 		     desc_idx++) {
-- 
2.7.4

[dpdk-dev] [RFC 6/6] virtio1.1: introduce the DESC_WB flag

[Tiwei Bie]

This patch introduces the DESC_WB flag, and adopts it in the virtio
Tx path. This flag can help to reduce the memory write operations on
the descriptor ring from the backend. So it could help to reduce the
PCIe transactions in a hardware backend case.

Normally the DESC_WB like mechanism is used in a head/tail design.
But in this prototype, head/tail isn't used. So backend still needs
to judge whether a desc is available by checking the DESC_HW flag.
Then some tricks are needed by the frontend when setting the DESC_WB
flags. Otherwise backend may see some descriptors' DESC_HW are set,
but actually they are not available to the backend.

Signed-off-by: Tiwei Bie <tiwei.bie@intel.com>
---
 drivers/net/virtio/virtio-1.1.h      |  1 +
 drivers/net/virtio/virtio_rxtx.c     | 14 +++++++++
 drivers/net/virtio/virtio_rxtx_1.1.c | 55 +++++++++++++++++++++++++-----------
 drivers/net/virtio/virtqueue.h       |  2 ++
 lib/librte_vhost/virtio-1.1.h        |  1 +
 lib/librte_vhost/virtio_net.c        | 37 +++++++++++++++---------
 6 files changed, 79 insertions(+), 31 deletions(-)

diff --git a/drivers/net/virtio/virtio-1.1.h b/drivers/net/virtio/virtio-1.1.h
index 48cbb18..bd76761 100644
--- a/drivers/net/virtio/virtio-1.1.h
+++ b/drivers/net/virtio/virtio-1.1.h
@@ -8,6 +8,7 @@
 #define BATCH_NOT_FIRST 0x0010
 #define BATCH_NOT_LAST  0x0020
 #define DESC_HW		0x0080
+#define DESC_WB		0x0100
 
 struct vring_desc_1_1 {
         uint64_t addr;
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 7be9aae..0248638 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -153,6 +153,10 @@ virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 #define DEFAULT_TX_FREE_THRESH 32
 #endif
 
+#ifndef DEFAULT_TX_RS_THRESH
+#define DEFAULT_TX_RS_THRESH 32
+#endif
+
 /* Cleanup from completed transmits. */
 static void
 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
@@ -434,6 +438,7 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 	struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
 	struct virtnet_tx *txvq;
 	uint16_t tx_free_thresh;
+	uint16_t tx_rs_thresh;
 	uint16_t desc_idx;
 
 	PMD_INIT_FUNC_TRACE();
@@ -451,6 +456,9 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 	if (tx_free_thresh == 0)
 		tx_free_thresh =
 			RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
+	tx_rs_thresh = tx_conf->tx_rs_thresh;
+	if (tx_rs_thresh == 0)
+		tx_rs_thresh = DEFAULT_TX_RS_THRESH;
 
 	if (tx_free_thresh >= (vq->vq_nentries - 3)) {
 		RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the "
@@ -461,7 +469,13 @@ virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
 		return -EINVAL;
 	}
 
+	if (!rte_is_power_of_2(tx_free_thresh)) {
+		RTE_LOG(ERR, PMD, "tx_free_thresh is not powerof 2");
+		return -EINVAL;
+	}
+
 	vq->vq_free_thresh = tx_free_thresh;
+	vq->vq_rs_thresh = tx_rs_thresh;
 
 	if (hw->use_simple_rxtx) {
 		uint16_t mid_idx  = vq->vq_nentries >> 1;
diff --git a/drivers/net/virtio/virtio_rxtx_1.1.c b/drivers/net/virtio/virtio_rxtx_1.1.c
index 883a027..38f6a4a 100644
--- a/drivers/net/virtio/virtio_rxtx_1.1.c
+++ b/drivers/net/virtio/virtio_rxtx_1.1.c
@@ -63,21 +63,28 @@
 #include "virtio_rxtx.h"
 
 /* Cleanup from completed transmits. */
-static void
+static inline int
 virtio_xmit_cleanup(struct virtqueue *vq)
 {
-	uint16_t idx;
-	uint16_t size = vq->vq_nentries;
+	uint16_t clean_to, idx;
+	uint16_t mask = vq->vq_nentries - 1;
 	struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
+	uint16_t last_cleaned = vq->vq_used_cons_idx - 1;
 
-	idx = vq->vq_used_cons_idx & (size - 1);
-	while ((desc[idx].flags & DESC_HW) == 0) {
-		idx = (++vq->vq_used_cons_idx) & (size - 1);
-		vq->vq_free_cnt++;
-
-		if (vq->vq_free_cnt >= size)
-			break;
+	clean_to = last_cleaned + vq->vq_rs_thresh;
+	if ((desc[clean_to & mask].flags & DESC_HW) != 0) {
+		PMD_TX_LOG(DEBUG, "TX descriptor %d is not done",
+			clean_to & mask);
+		return -1;
 	}
+
+	for (idx = last_cleaned + 2; idx < clean_to; idx++)
+		desc[idx & mask].flags &= ~DESC_HW;
+
+	vq->vq_used_cons_idx = clean_to + 1;
+	vq->vq_free_cnt += vq->vq_rs_thresh;
+
+	return 0;
 }
 
 uint16_t
@@ -90,6 +97,7 @@ virtio_xmit_pkts_1_1(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts
 	struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
 	uint16_t idx;
 	struct vq_desc_extra *dxp;
+	uint16_t nb_needed, nb_used = vq->vq_nb_used;
 
 	if (unlikely(nb_pkts < 1))
 		return nb_pkts;
@@ -103,19 +111,23 @@ virtio_xmit_pkts_1_1(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts
 		struct rte_mbuf *txm = tx_pkts[i];
 		struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
 
-		if (unlikely(txm->nb_segs + 1 > vq->vq_free_cnt)) {
-			virtio_xmit_cleanup(vq);
+		nb_needed = txm->nb_segs + 1;
 
-			if (unlikely(txm->nb_segs + 1 > vq->vq_free_cnt)) {
-				PMD_TX_LOG(ERR,
-					   "No free tx descriptors to transmit");
-				break;
+		if (unlikely(nb_needed > vq->vq_free_cnt)) {
+			if (unlikely(virtio_xmit_cleanup(vq) != 0))
+				goto end_of_tx;
+
+			if (unlikely(nb_needed > vq->vq_rs_thresh)) {
+				while (nb_needed > vq->vq_free_cnt) {
+					if (virtio_xmit_cleanup(vq) != 0)
+						goto end_of_tx;
+				}
 			}
 		}
 
 		txvq->stats.bytes += txm->pkt_len;
 
-		vq->vq_free_cnt -= txm->nb_segs + 1;
+		vq->vq_free_cnt -= nb_needed;
 
 		idx = (vq->vq_avail_idx++) & (vq->vq_nentries - 1);
 		dxp = &vq->vq_descx[idx];
@@ -129,22 +141,31 @@ virtio_xmit_pkts_1_1(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts
 		desc[idx].flags = VRING_DESC_F_NEXT;
 		if (i != 0)
 			desc[idx].flags |= DESC_HW;
+		nb_used = (nb_used + 1) & ~vq->vq_rs_thresh;
+		if (nb_used == 0 || nb_used == 1)
+			desc[idx].flags |= DESC_WB;
 
 		do {
 			idx = (vq->vq_avail_idx++) & (vq->vq_nentries - 1);
 			desc[idx].addr  = VIRTIO_MBUF_DATA_DMA_ADDR(txm, vq);
 			desc[idx].len   = txm->data_len;
 			desc[idx].flags = DESC_HW | VRING_DESC_F_NEXT;
+			nb_used = (nb_used + 1) & ~vq->vq_rs_thresh;
+			if (nb_used == 0 || nb_used == 1)
+				desc[idx].flags |= DESC_WB;
 		} while ((txm = txm->next) != NULL);
 
 		desc[idx].flags &= ~VRING_DESC_F_NEXT;
 	}
 
+end_of_tx:
 	if (likely(i)) {
 		rte_smp_wmb();
 		vq->vq_ring.desc_1_1[head_idx & (vq->vq_nentries - 1)].flags |= DESC_HW;
 	}
 
+	vq->vq_nb_used = nb_used;
+
 	txvq->stats.packets += i;
 	txvq->stats.errors  += nb_pkts - i;
 
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 45f49d7..b104afa 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -194,6 +194,8 @@ struct virtqueue {
 	uint16_t vq_free_cnt;  /**< num of desc available */
 	uint16_t vq_avail_idx; /**< sync until needed */
 	uint16_t vq_free_thresh; /**< free threshold */
+	uint16_t vq_rs_thresh; /**< RS threshold */
+	uint16_t vq_nb_used ;  /**< num of unreported desc */
 
 	void *vq_ring_virt_mem;  /**< linear address of vring*/
 	unsigned int vq_ring_size;
diff --git a/lib/librte_vhost/virtio-1.1.h b/lib/librte_vhost/virtio-1.1.h
index 4241d0a..91e21f1 100644
--- a/lib/librte_vhost/virtio-1.1.h
+++ b/lib/librte_vhost/virtio-1.1.h
@@ -12,6 +12,7 @@
 #define BATCH_NOT_FIRST 0x0010
 #define BATCH_NOT_LAST  0x0020
 #define DESC_HW		0x0080
+#define DESC_WB		0x0100
 
 struct vring_desc_1_1 {
         __le64 addr;
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 08d53d9..96b84e5 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -1100,7 +1100,8 @@ mbuf_is_consumed(struct rte_mbuf *m)
 static inline uint16_t __attribute__((always_inline))
 dequeue_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	     struct rte_mempool *mbuf_pool, struct rte_mbuf *m,
-	     struct vring_desc_1_1 *descs, uint16_t *desc_idx)
+	     struct vring_desc_1_1 *descs, uint16_t *desc_idx,
+	     struct vring_desc_1_1 **dps, uint16_t *_cnt)
 {
 	struct vring_desc_1_1 *desc;
 	uint64_t desc_addr;
@@ -1110,6 +1111,7 @@ dequeue_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct rte_mbuf *cur = m, *prev = m;
 	struct virtio_net_hdr *hdr = NULL;
 	uint16_t head_idx = *desc_idx;
+	uint16_t cnt = *_cnt;
 
 	desc = &descs[(head_idx++) & (vq->size - 1)];
 	if (unlikely((desc->len < dev->vhost_hlen)) ||
@@ -1132,6 +1134,9 @@ dequeue_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	 */
 	if (likely((desc->len == dev->vhost_hlen) &&
 		   (desc->flags & VRING_DESC_F_NEXT) != 0)) {
+		if (desc->flags & DESC_WB)
+			dps[cnt++] = desc;
+
 		desc = &descs[(head_idx++) & (vq->size - 1)];
 		if (unlikely(desc->flags & VRING_DESC_F_INDIRECT))
 			return -1;
@@ -1172,6 +1177,9 @@ dequeue_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if ((desc->flags & VRING_DESC_F_NEXT) == 0)
 				break;
 
+			if (desc->flags & DESC_WB)
+				dps[cnt++] = desc;
+
 			desc = &descs[(head_idx++) & (vq->size - 1)];
 			if (unlikely(desc->flags & VRING_DESC_F_INDIRECT))
 				return -1;
@@ -1217,7 +1225,11 @@ dequeue_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(hdr, m);
 
+	if (desc->flags & DESC_WB)
+		dps[cnt++] = desc;
+
 	*desc_idx = head_idx;
+	*_cnt = cnt;
 
 	return 0;
 }
@@ -1229,7 +1241,8 @@ vhost_dequeue_burst_1_1(struct virtio_net *dev, struct vhost_virtqueue *vq,
 {
 	uint16_t i;
 	struct vring_desc_1_1 *desc = vq->desc_1_1;
-	uint16_t head_idx = vq->last_used_idx;
+	struct vring_desc_1_1 *dps[64];
+	uint16_t cnt = 0;
 	uint16_t desc_idx;
 	int err;
 
@@ -1250,26 +1263,22 @@ vhost_dequeue_burst_1_1(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			break;
 		}
 
-		err = dequeue_desc(dev, vq, mbuf_pool, pkts[i], desc, &desc_idx);
+		err = dequeue_desc(dev, vq, mbuf_pool, pkts[i], desc,
+				   &desc_idx, dps, &cnt);
 		if (unlikely(err)) {
 			rte_pktmbuf_free(pkts[i]);
 			break;
 		}
 	}
+	count = i;
 
 	vq->last_used_idx = desc_idx;
-	if (likely(i)) {
-		rte_prefetch0(&desc[head_idx & (vq->size - 1)]);
-		for (desc_idx = head_idx + 1;
-		     desc_idx != vq->last_used_idx;
-		     desc_idx++) {
-			desc[desc_idx & (vq->size - 1)].flags = 0;
-		}
-		rte_smp_wmb();
-		desc[head_idx & (vq->size - 1)].flags = 0;
-	}
 
-	return i;
+	rte_smp_wmb();
+	for (i = 0; i < cnt; i++)
+		dps[i]->flags = 0;
+
+	return count;
 }
 
 uint16_t
-- 
2.7.4