[dpdk-dev] [PATCH 0/3] net/virtio: Rx paths cleanup

[dpdk-dev] [PATCH 0/3] net/virtio: Rx paths cleanup

[Maxime Coquelin]

This is series mainly merges out-of-order mergeable and
non-mergeable Rx paths.

Doing so, we remove one path, and micro-benchmark does not
show any performance regression:

                +---------+----------+----------+----------+----------+----------+
                |            V18.11             |    v18.11 + Rx paths merge     |
                +---------+----------+----------+----------+----------+----------+
                | 1 queue | 2 queues | 3 queues | 1 queue  | 2 queues | 3 queues |
+-----+---------+---------+----------+----------+----------+----------+----------+
| MRG | Rxonly  | 16.22   | 32.47    | 48.81    | 16.31    | 32.62    | 48.88    |
| OFF | IO loop | 12.86   | 25.69    | 38.16    | 12.88    | 25.74    | 38.53    |
+-----+---------+---------+----------+----------+----------+----------+----------+
| MRG | Rxonly  | 16.27   | 32.65    | 48.81    | 16.29    | 32.67    | 48.77    |
| ON  | IO loop | 12.82   | 25.59    | 37.86    | 12.86    | 25.68    | 37.9     |
+-----+---------+---------+----------+----------+----------+----------+----------+

Note that to avoid vector path to be selected as this benchmark is done
without offloads, I had to do this small change to disable it in
virtio_dev_configure():
-       hw->use_simple_rx = 1;
+       hw->use_simple_rx = 0;

The series also enables in-order path selection when mergeable buffers feature
is disabled. Offload and refill helpers are also changed to be inlined.

Maxime Coquelin (3):
  net/virtio: inline refill and offload helpers
  net/virtio: merge Rx mergeable and non-mergeable paths
  net/virtio: add non-mergeable support to in-order path

 drivers/net/virtio/virtio_ethdev.c |  16 +---
 drivers/net/virtio/virtio_ethdev.h |   5 +-
 drivers/net/virtio/virtio_rxtx.c   | 133 ++++-------------------------
 3 files changed, 22 insertions(+), 132 deletions(-)

-- 
2.17.2

[dpdk-dev] [PATCH 1/3] net/virtio: inline refill and offload helpers

[Maxime Coquelin]

Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/net/virtio/virtio_rxtx.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index eb891433e..e1c270b1c 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -741,7 +741,7 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
 	return 0;
 }
 
-static void
+static inline void
 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 {
 	int error;
@@ -757,7 +757,7 @@ virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
 	}
 }
 
-static void
+static inline void
 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m)
 {
 	int error;
@@ -769,7 +769,7 @@ virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m)
 	}
 }
 
-static void
+static inline void
 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
 {
 	uint32_t s = mbuf->pkt_len;
@@ -811,7 +811,7 @@ virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m)
 }
 
 /* Optionally fill offload information in structure */
-static int
+static inline int
 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
 {
 	struct rte_net_hdr_lens hdr_lens;
-- 
2.17.2

[dpdk-dev] [PATCH 2/3] net/virtio: merge Rx mergeable and non-mergeable paths

[Maxime Coquelin]

Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/net/virtio/virtio_ethdev.c |   5 --
 drivers/net/virtio/virtio_ethdev.h |   3 -
 drivers/net/virtio/virtio_rxtx.c   | 115 ++---------------------------
 3 files changed, 7 insertions(+), 116 deletions(-)

diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
index 2ba66d291..9658b179a 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1335,11 +1335,6 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
 			"virtio: using inorder mergeable buffer Rx path on port %u",
 			eth_dev->data->port_id);
 		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts_inorder;
-	} else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
-		PMD_INIT_LOG(INFO,
-			"virtio: using mergeable buffer Rx path on port %u",
-			eth_dev->data->port_id);
-		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
 	} else {
 		PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
 			eth_dev->data->port_id);
diff --git a/drivers/net/virtio/virtio_ethdev.h b/drivers/net/virtio/virtio_ethdev.h
index e0f80e5a4..865863300 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -74,9 +74,6 @@ int virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
 uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts);
 
-uint16_t virtio_recv_mergeable_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
-		uint16_t nb_pkts);
-
 uint16_t virtio_recv_mergeable_pkts_inorder(void *rx_queue,
 		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
 
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index e1c270b1c..331c1c56d 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -883,111 +883,6 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
 }
 
 #define VIRTIO_MBUF_BURST_SZ 64
-#define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
-uint16_t
-virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
-{
-	struct virtnet_rx *rxvq = rx_queue;
-	struct virtqueue *vq = rxvq->vq;
-	struct virtio_hw *hw = vq->hw;
-	struct rte_mbuf *rxm, *new_mbuf;
-	uint16_t nb_used, num, nb_rx;
-	uint32_t len[VIRTIO_MBUF_BURST_SZ];
-	struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
-	int error;
-	uint32_t i, nb_enqueued;
-	uint32_t hdr_size;
-	struct virtio_net_hdr *hdr;
-
-	nb_rx = 0;
-	if (unlikely(hw->started == 0))
-		return nb_rx;
-
-	nb_used = VIRTQUEUE_NUSED(vq);
-
-	virtio_rmb();
-
-	num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
-	if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
-		num = VIRTIO_MBUF_BURST_SZ;
-	if (likely(num > DESC_PER_CACHELINE))
-		num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
-
-	num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
-	PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
-
-	nb_enqueued = 0;
-	hdr_size = hw->vtnet_hdr_size;
-
-	for (i = 0; i < num ; i++) {
-		rxm = rcv_pkts[i];
-
-		PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
-
-		if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) {
-			PMD_RX_LOG(ERR, "Packet drop");
-			nb_enqueued++;
-			virtio_discard_rxbuf(vq, rxm);
-			rxvq->stats.errors++;
-			continue;
-		}
-
-		rxm->port = rxvq->port_id;
-		rxm->data_off = RTE_PKTMBUF_HEADROOM;
-		rxm->ol_flags = 0;
-		rxm->vlan_tci = 0;
-
-		rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
-		rxm->data_len = (uint16_t)(len[i] - hdr_size);
-
-		hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
-			RTE_PKTMBUF_HEADROOM - hdr_size);
-
-		if (hw->vlan_strip)
-			rte_vlan_strip(rxm);
-
-		if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
-			virtio_discard_rxbuf(vq, rxm);
-			rxvq->stats.errors++;
-			continue;
-		}
-
-		virtio_rx_stats_updated(rxvq, rxm);
-
-		rx_pkts[nb_rx++] = rxm;
-	}
-
-	rxvq->stats.packets += nb_rx;
-
-	/* Allocate new mbuf for the used descriptor */
-	while (likely(!virtqueue_full(vq))) {
-		new_mbuf = rte_mbuf_raw_alloc(rxvq->mpool);
-		if (unlikely(new_mbuf == NULL)) {
-			struct rte_eth_dev *dev
-				= &rte_eth_devices[rxvq->port_id];
-			dev->data->rx_mbuf_alloc_failed++;
-			break;
-		}
-		error = virtqueue_enqueue_recv_refill(vq, new_mbuf);
-		if (unlikely(error)) {
-			rte_pktmbuf_free(new_mbuf);
-			break;
-		}
-		nb_enqueued++;
-	}
-
-	if (likely(nb_enqueued)) {
-		vq_update_avail_idx(vq);
-
-		if (unlikely(virtqueue_kick_prepare(vq))) {
-			virtqueue_notify(vq);
-			PMD_RX_LOG(DEBUG, "Notified");
-		}
-	}
-
-	return nb_rx;
-}
-
 uint16_t
 virtio_recv_mergeable_pkts_inorder(void *rx_queue,
 			struct rte_mbuf **rx_pkts,
@@ -1176,7 +1071,7 @@ virtio_recv_mergeable_pkts_inorder(void *rx_queue,
 }
 
 uint16_t
-virtio_recv_mergeable_pkts(void *rx_queue,
+virtio_recv_pkts(void *rx_queue,
 			struct rte_mbuf **rx_pkts,
 			uint16_t nb_pkts)
 {
@@ -1239,10 +1134,14 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 
 		header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr +
 			RTE_PKTMBUF_HEADROOM - hdr_size);
-		seg_num = header->num_buffers;
 
-		if (seg_num == 0)
+		if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+			seg_num = header->num_buffers;
+			if (seg_num == 0)
+				seg_num = 1;
+		} else {
 			seg_num = 1;
+		}
 
 		rxm->data_off = RTE_PKTMBUF_HEADROOM;
 		rxm->nb_segs = seg_num;
-- 
2.17.2

[dpdk-dev] [PATCH 3/3] net/virtio: add non-mergeable support to in-order path

[Maxime Coquelin]

Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/net/virtio/virtio_ethdev.c | 11 +++--------
 drivers/net/virtio/virtio_ethdev.h |  2 +-
 drivers/net/virtio/virtio_rxtx.c   | 10 +++++++---
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
index 9658b179a..6670bf1ec 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1332,9 +1332,9 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
 		eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
 	} else if (hw->use_inorder_rx) {
 		PMD_INIT_LOG(INFO,
-			"virtio: using inorder mergeable buffer Rx path on port %u",
+			"virtio: using in-order Rx path on port %u",
 			eth_dev->data->port_id);
-		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts_inorder;
+		eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder;
 	} else {
 		PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u",
 			eth_dev->data->port_id);
@@ -1901,12 +1901,7 @@ virtio_dev_configure(struct rte_eth_dev *dev)
 
 	if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
 		hw->use_inorder_tx = 1;
-		if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
-			hw->use_inorder_rx = 1;
-			hw->use_simple_rx = 0;
-		} else {
-			hw->use_inorder_rx = 0;
-		}
+		hw->use_inorder_rx = 1;
 	}
 
 #if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM
diff --git a/drivers/net/virtio/virtio_ethdev.h b/drivers/net/virtio/virtio_ethdev.h
index 865863300..96b4928e9 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -74,7 +74,7 @@ int virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
 uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts);
 
-uint16_t virtio_recv_mergeable_pkts_inorder(void *rx_queue,
+uint16_t virtio_recv_pkts_inorder(void *rx_queue,
 		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
 
 uint16_t virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 331c1c56d..ab472b01f 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -884,7 +884,7 @@ virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
 
 #define VIRTIO_MBUF_BURST_SZ 64
 uint16_t
-virtio_recv_mergeable_pkts_inorder(void *rx_queue,
+virtio_recv_pkts_inorder(void *rx_queue,
 			struct rte_mbuf **rx_pkts,
 			uint16_t nb_pkts)
 {
@@ -941,10 +941,14 @@ virtio_recv_mergeable_pkts_inorder(void *rx_queue,
 		header = (struct virtio_net_hdr_mrg_rxbuf *)
 			 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
 			 - hdr_size);
-		seg_num = header->num_buffers;
 
-		if (seg_num == 0)
+		if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+			seg_num = header->num_buffers;
+			if (seg_num == 0)
+				seg_num = 1;
+		} else {
 			seg_num = 1;
+		}
 
 		rxm->data_off = RTE_PKTMBUF_HEADROOM;
 		rxm->nb_segs = seg_num;
-- 
2.17.2

Re: [dpdk-dev] [PATCH 2/3] net/virtio: merge Rx mergeable and non-mergeable paths

[Maxime Coquelin]

On 12/3/18 4:10 PM, Maxime Coquelin wrote:
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
>   drivers/net/virtio/virtio_ethdev.c |   5 --
>   drivers/net/virtio/virtio_ethdev.h |   3 -
>   drivers/net/virtio/virtio_rxtx.c   | 115 ++---------------------------
>   3 files changed, 7 insertions(+), 116 deletions(-)


Intel STV team ran more tests and found a performance regression.
I managed to reproduce it, and worked on optimizing the Rx path.
I managed to go from -25% to -6% for this use case, but this is still a
significant regression so I'll drop this patch.

The good news is that the optimization is valid for mergeable buffers
case, and it gains 5% so I'll post this optimization instead.

Thanks to the STV team for the testing,
Maxime