[dpdk-dev] [PATCH 0/5] net/mlx5: add vectorized Rx/Tx burst for x86

[dpdk-dev] [PATCH 0/5] net/mlx5: add vectorized Rx/Tx burst for x86

[Yongseok Koh]

This is to introduce more efficient Rx/Tx burst functions using SIMD
instructions. Currently it is only supported by 64bit x86 having SSE4.1.

>From functional perspective, Rx burst function is equivalent to the
existing mlx5_rx_burst() except for scatter support, which will be added
soon. Tx burst function supports multi-segment packets and offload flags
unless it is disabled by txq_flags. However, disabling those features will
bring a little higher performance.

Yongseok Koh (5):
  net/mlx5: change indexing for Tx SW ring
  net/mlx5: free buffers in bulk on Tx completion
  net/mlx5: use buffer address for LKEY search
  net/mlx5: select Rx/Tx callbacks when starting device
  net/mlx5: add vectorized Rx/Tx burst for SSE4.1

 drivers/net/mlx5/Makefile            |   10 +
 drivers/net/mlx5/mlx5_defs.h         |   18 +
 drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
 drivers/net/mlx5/mlx5_mr.c           |    2 +
 drivers/net/mlx5/mlx5_rxq.c          |   57 +-
 drivers/net/mlx5/mlx5_rxtx.c         |  425 +++-------
 drivers/net/mlx5/mlx5_rxtx.h         |  293 ++++++-
 drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_trigger.c      |    3 +
 drivers/net/mlx5/mlx5_txq.c          |   20 +-
 10 files changed, 1970 insertions(+), 337 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c

-- 
2.11.0

[dpdk-dev] [PATCH 1/5] net/mlx5: change indexing for Tx SW ring

[Yongseok Koh]

For Tx SW ring (txq->elts[]), indexes are kept and used in
txq->elts_head/tail. Because of this, one entry must always be left unused
and it also makes code complex. Changed to store counters instead of
indexes in order to make the code simpler and to reduce a few calculations.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 108 +++++++++++++++++--------------------------
 drivers/net/mlx5/mlx5_rxtx.h |   4 +-
 drivers/net/mlx5/mlx5_txq.c  |  16 +++----
 3 files changed, 52 insertions(+), 76 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 70314b393..43db06ad8 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -255,7 +255,8 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
 static inline void
 txq_complete(struct txq *txq)
 {
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	const unsigned int cqe_n = 1 << txq->cqe_n;
 	const unsigned int cqe_cnt = cqe_n - 1;
 	uint16_t elts_free = txq->elts_tail;
@@ -292,24 +293,23 @@ txq_complete(struct txq *txq)
 	ctrl = (volatile struct mlx5_wqe_ctrl *)
 		tx_mlx5_wqe(txq, txq->wqe_pi);
 	elts_tail = ctrl->ctrl3;
-	assert(elts_tail < (1 << txq->wqe_n));
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
 	/* Free buffers. */
 	while (elts_free != elts_tail) {
-		struct rte_mbuf *elt = (*txq->elts)[elts_free];
-		unsigned int elts_free_next =
-			(elts_free + 1) & (elts_n - 1);
-		struct rte_mbuf *elt_next = (*txq->elts)[elts_free_next];
+		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
+		struct rte_mbuf *elt_next =
+			(*txq->elts)[(elts_free + 1) & elts_m];
 
 #ifndef NDEBUG
 		/* Poisoning. */
-		memset(&(*txq->elts)[elts_free],
+		memset(&(*txq->elts)[elts_free & elts_m],
 		       0x66,
-		       sizeof((*txq->elts)[elts_free]));
+		       sizeof((*txq->elts)[elts_free & elts_m]));
 #endif
 		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
 		/* Only one segment needs to be freed. */
 		rte_pktmbuf_free_seg(elt);
-		elts_free = elts_free_next;
+		++elts_free;
 	}
 	txq->cq_ci = cq_ci;
 	txq->elts_tail = elts_tail;
@@ -409,12 +409,10 @@ int
 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
 {
 	struct txq *txq = tx_queue;
-	const unsigned int elts_n = 1 << txq->elts_n;
-	const unsigned int elts_cnt = elts_n - 1;
-	unsigned int used;
+	uint16_t used;
 
 	txq_complete(txq);
-	used = (txq->elts_head - txq->elts_tail) & elts_cnt;
+	used = txq->elts_head - txq->elts_tail;
 	if (offset < used)
 		return RTE_ETH_TX_DESC_FULL;
 	return RTE_ETH_TX_DESC_DONE;
@@ -488,11 +486,12 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
 	unsigned int k = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	unsigned int max_inline = txq->max_inline;
 	const unsigned int inline_en = !!max_inline && txq->inline_en;
 	uint16_t max_wqe;
@@ -509,9 +508,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(*pkts);
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
@@ -540,9 +537,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--segs_n;
 		if (unlikely(--max_wqe == 0))
 			break;
@@ -561,7 +558,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		if (length < (MLX5_WQE_DWORD_SIZE + 2))
 			break;
 		/* Update element. */
-		(*txq->elts)[elts_head] = buf;
+		(*txq->elts)[elts_head & elts_m] = buf;
 		/* Prefetch next buffer data. */
 		if (pkts_n - i > 1)
 			rte_prefetch0(
@@ -801,8 +798,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr,
 			naddr >> 32,
 		};
-		elts_head = (elts_head + 1) & (elts_n - 1);
-		(*txq->elts)[elts_head] = buf;
+		(*txq->elts)[++elts_head & elts_m] = buf;
 		++sg;
 		/* Advance counter only if all segs are successfully posted. */
 		if (sg < segs_n)
@@ -810,7 +806,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		else
 			j += sg;
 next_pkt:
-		elts_head = (elts_head + 1) & (elts_n - 1);
+		++elts_head;
 		++pkts;
 		++i;
 		/* Initialize known and common part of the WQE structure. */
@@ -853,7 +849,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Take a shortcut if nothing must be sent. */
 	if (unlikely((i + k) == 0))
 		return 0;
-	txq->elts_head = (txq->elts_head + i + j) & (elts_n - 1);
+	txq->elts_head += (i + j);
 	/* Check whether completion threshold has been reached. */
 	comp = txq->elts_comp + i + j + k;
 	if (comp >= MLX5_TX_COMP_THRESH) {
@@ -960,10 +956,11 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int comp;
 	struct mlx5_mpw mpw = {
@@ -977,15 +974,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
 		uint32_t cs_flags = 0;
@@ -995,12 +989,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--pkts_n;
 		/* Should we enable HW CKSUM offload */
 		if (buf->ol_flags &
@@ -1036,9 +1030,8 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			volatile struct mlx5_wqe_data_seg *dseg;
 			uintptr_t addr;
 
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
 			assert(buf);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			dseg = mpw.data.dseg[mpw.pkts_n];
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
@@ -1046,7 +1039,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 				.addr = htonll(addr),
 			};
-			elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 			length += DATA_LEN(buf);
 #endif
@@ -1057,7 +1049,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		assert(length == mpw.len);
 		if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
 			mlx5_mpw_close(txq, &mpw);
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
@@ -1175,10 +1166,11 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int comp;
 	unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE;
@@ -1205,12 +1197,9 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uintptr_t addr;
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
@@ -1221,12 +1210,12 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--pkts_n;
 		/*
 		 * Compute max_wqe in case less WQE were consumed in previous
@@ -1287,10 +1276,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 			do {
 				volatile struct mlx5_wqe_data_seg *dseg;
 
-				elts_head_next =
-					(elts_head + 1) & (elts_n - 1);
 				assert(buf);
-				(*txq->elts)[elts_head] = buf;
+				(*txq->elts)[elts_head++ & elts_m] = buf;
 				dseg = mpw.data.dseg[mpw.pkts_n];
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
@@ -1298,7 +1285,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 					.addr = htonll(addr),
 				};
-				elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 				length += DATA_LEN(buf);
 #endif
@@ -1315,9 +1301,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 			assert(mpw.state == MLX5_MPW_INL_STATE_OPENED);
 			assert(length <= inline_room);
 			assert(length == DATA_LEN(buf));
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			/* Maximum number of bytes before wrapping. */
 			max = ((((uintptr_t)(txq->wqes)) +
 				(1 << txq->wqe_n) *
@@ -1354,7 +1339,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				inline_room -= length;
 			}
 		}
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
@@ -1476,10 +1460,11 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max_elts;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE;
 	unsigned int mpw_room = 0;
@@ -1494,8 +1479,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Start processing. */
 	txq_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	if (max_elts > elts_n)
-		max_elts -= elts_n;
 	/* A CQE slot must always be available. */
 	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
@@ -1503,7 +1486,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		return 0;
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uintptr_t addr;
 		uint64_t naddr;
 		unsigned int n;
@@ -1517,7 +1499,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max_elts - j < segs_n + 1)
+		if (max_elts - j < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
@@ -1601,10 +1583,8 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			do {
 				volatile struct mlx5_wqe_data_seg *dseg;
 
-				elts_head_next =
-					(elts_head + 1) & (elts_n - 1);
 				assert(buf);
-				(*txq->elts)[elts_head] = buf;
+				(*txq->elts)[elts_head++ & elts_m] = buf;
 				dseg = mpw.data.dseg[mpw.pkts_n];
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
@@ -1612,7 +1592,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 					.addr = htonll(addr),
 				};
-				elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 				length += DATA_LEN(buf);
 #endif
@@ -1663,7 +1642,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			/* No need to get completion as the entire packet is
 			 * copied to WQ. Free the buf right away.
 			 */
-			elts_head_next = elts_head;
 			rte_pktmbuf_free_seg(buf);
 			mpw_room -= (inl_pad + sizeof(inl_hdr) + length);
 			/* Add pad in the next packet if any. */
@@ -1686,8 +1664,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				dseg = (volatile void *)
 					((uintptr_t)mpw.data.raw +
 					 inl_pad);
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			for (n = 0; n * RTE_CACHE_LINE_SIZE < length; n++)
 				rte_prefetch2((void *)(addr +
@@ -1706,7 +1683,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			mpw_room -= (inl_pad + sizeof(*dseg));
 			inl_pad = 0;
 		}
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 450a56966..f9b738b4e 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -242,8 +242,8 @@ struct hash_rxq {
 /* TX queue descriptor. */
 RTE_STD_C11
 struct txq {
-	uint16_t elts_head; /* Current index in (*elts)[]. */
-	uint16_t elts_tail; /* First element awaiting completion. */
+	uint16_t elts_head; /* Current counter in (*elts)[]. */
+	uint16_t elts_tail; /* Counter of first element awaiting completion. */
 	uint16_t elts_comp; /* Counter since last completion request. */
 	uint16_t mpw_comp; /* WQ index since last completion request. */
 	uint16_t cq_ci; /* Consumer index for completion queue. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index bf72468d5..f0729a2a8 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -103,9 +103,10 @@ txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n)
 static void
 txq_free_elts(struct txq_ctrl *txq_ctrl)
 {
-	unsigned int elts_n = 1 << txq_ctrl->txq.elts_n;
-	unsigned int elts_head = txq_ctrl->txq.elts_head;
-	unsigned int elts_tail = txq_ctrl->txq.elts_tail;
+	const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	uint16_t elts_head = txq_ctrl->txq.elts_head;
+	uint16_t elts_tail = txq_ctrl->txq.elts_tail;
 	struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts;
 
 	DEBUG("%p: freeing WRs", (void *)txq_ctrl);
@@ -114,18 +115,17 @@ txq_free_elts(struct txq_ctrl *txq_ctrl)
 	txq_ctrl->txq.elts_comp = 0;
 
 	while (elts_tail != elts_head) {
-		struct rte_mbuf *elt = (*elts)[elts_tail];
+		struct rte_mbuf *elt = (*elts)[elts_tail & elts_m];
 
 		assert(elt != NULL);
 		rte_pktmbuf_free_seg(elt);
 #ifndef NDEBUG
 		/* Poisoning. */
-		memset(&(*elts)[elts_tail],
+		memset(&(*elts)[elts_tail & elts_m],
 		       0x77,
-		       sizeof((*elts)[elts_tail]));
+		       sizeof((*elts)[elts_tail & elts_m]));
 #endif
-		if (++elts_tail == elts_n)
-			elts_tail = 0;
+		++elts_tail;
 	}
 }
 
-- 
2.11.0

Re: [dpdk-dev] [PATCH 1/5] net/mlx5: change indexing for Tx SW ring

[Nélio Laranjeiro]

On Wed, Jun 28, 2017 at 04:03:59PM -0700, Yongseok Koh wrote:
> For Tx SW ring (txq->elts[]), indexes are kept and used in
> txq->elts_head/tail. Because of this, one entry must always be left unused
> and it also makes code complex. Changed to store counters instead of
> indexes in order to make the code simpler and to reduce a few calculations.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH 2/5] net/mlx5: free buffers in bulk on Tx completion

[Yongseok Koh]

When processing Tx completion, it is more efficient to free buffers in bulk
using rte_mempool_put_bulk() if buffers are from a same mempool.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 43db06ad8..d81d630f7 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -264,6 +264,8 @@ txq_complete(struct txq *txq)
 	uint16_t cq_ci = txq->cq_ci;
 	volatile struct mlx5_cqe *cqe = NULL;
 	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	unsigned int blk_n = 0;
 
 	do {
 		volatile struct mlx5_cqe *tmp;
@@ -296,21 +298,37 @@ txq_complete(struct txq *txq)
 	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
 	/* Free buffers. */
 	while (elts_free != elts_tail) {
-		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
-		struct rte_mbuf *elt_next =
-			(*txq->elts)[(elts_free + 1) & elts_m];
-
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (blk_n) {
+				if (likely(m->pool == free[0]->pool)) {
+					free[blk_n++] = m;
+				} else {
+					rte_mempool_put_bulk(
+							free[0]->pool,
+							(void *)free,
+							blk_n);
+					free[0] = m;
+					blk_n = 1;
+				}
+			} else {
+				free[0] = m;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(free[0]->pool, (void *)free, blk_n);
 #ifndef NDEBUG
-		/* Poisoning. */
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
 		memset(&(*txq->elts)[elts_free & elts_m],
 		       0x66,
 		       sizeof((*txq->elts)[elts_free & elts_m]));
-#endif
-		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
-		/* Only one segment needs to be freed. */
-		rte_pktmbuf_free_seg(elt);
 		++elts_free;
 	}
+#endif
 	txq->cq_ci = cq_ci;
 	txq->elts_tail = elts_tail;
 	/* Update the consumer index. */
-- 
2.11.0

Re: [dpdk-dev] [PATCH 2/5] net/mlx5: free buffers in bulk on Tx completion

[Nélio Laranjeiro]

On Wed, Jun 28, 2017 at 04:04:00PM -0700, Yongseok Koh wrote:
> When processing Tx completion, it is more efficient to free buffers in bulk
> using rte_mempool_put_bulk() if buffers are from a same mempool.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5_rxtx.c | 36 +++++++++++++++++++++++++++---------
>  1 file changed, 27 insertions(+), 9 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> index 43db06ad8..d81d630f7 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -264,6 +264,8 @@ txq_complete(struct txq *txq)
>  	uint16_t cq_ci = txq->cq_ci;
>  	volatile struct mlx5_cqe *cqe = NULL;
>  	volatile struct mlx5_wqe_ctrl *ctrl;
> +	struct rte_mbuf *m, *free[elts_n];
> +	unsigned int blk_n = 0;
>  
>  	do {
>  		volatile struct mlx5_cqe *tmp;
> @@ -296,21 +298,37 @@ txq_complete(struct txq *txq)
>  	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
>  	/* Free buffers. */
>  	while (elts_free != elts_tail) {
> -		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
> -		struct rte_mbuf *elt_next =
> -			(*txq->elts)[(elts_free + 1) & elts_m];
> -
> +		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
> +		if (likely(m != NULL)) {
> +			if (blk_n) {
> +				if (likely(m->pool == free[0]->pool)) {
> +					free[blk_n++] = m;
> +				} else {
> +					rte_mempool_put_bulk(
> +							free[0]->pool,
> +							(void *)free,
> +							blk_n);

The indentation is strange here, free[0] should be on the same line as
rte_mempool_put_bulk.

> +					free[0] = m;
> +					blk_n = 1;
> +				}
> +			} else {
> +				free[0] = m;
> +				blk_n = 1;
> +			}
> +		}

This loop could be smaller, blk_n can only be equal to 0 in the first
iteration, otherwise is >= 1.
The first if statement can be merged with the second one: 

	if (likely(m != NULL)) {
		if (likely(blk_n && m->pool == free[0]->pool)) {
		...
		} else {
		...
		}

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH 2/5] net/mlx5: free buffers in bulk on Tx completion

[Nélio Laranjeiro]

On Fri, Jun 30, 2017 at 02:30:47PM +0200, Nélio Laranjeiro wrote:
> On Wed, Jun 28, 2017 at 04:04:00PM -0700, Yongseok Koh wrote:
> > When processing Tx completion, it is more efficient to free buffers in bulk
> > using rte_mempool_put_bulk() if buffers are from a same mempool.
> > 
> > Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> > ---
> >  drivers/net/mlx5/mlx5_rxtx.c | 36 +++++++++++++++++++++++++++---------
> >  1 file changed, 27 insertions(+), 9 deletions(-)
> > 
> > diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> > index 43db06ad8..d81d630f7 100644
> > --- a/drivers/net/mlx5/mlx5_rxtx.c
> > +++ b/drivers/net/mlx5/mlx5_rxtx.c
> > @@ -264,6 +264,8 @@ txq_complete(struct txq *txq)
> >  	uint16_t cq_ci = txq->cq_ci;
> >  	volatile struct mlx5_cqe *cqe = NULL;
> >  	volatile struct mlx5_wqe_ctrl *ctrl;
> > +	struct rte_mbuf *m, *free[elts_n];
> > +	unsigned int blk_n = 0;
> >  
> >  	do {
> >  		volatile struct mlx5_cqe *tmp;
> > @@ -296,21 +298,37 @@ txq_complete(struct txq *txq)
> >  	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
> >  	/* Free buffers. */
> >  	while (elts_free != elts_tail) {
> > -		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
> > -		struct rte_mbuf *elt_next =
> > -			(*txq->elts)[(elts_free + 1) & elts_m];
> > -
> > +		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
> > +		if (likely(m != NULL)) {
> > +			if (blk_n) {
> > +				if (likely(m->pool == free[0]->pool)) {
> > +					free[blk_n++] = m;
> > +				} else {
> > +					rte_mempool_put_bulk(
> > +							free[0]->pool,
> > +							(void *)free,
> > +							blk_n);
> 
> The indentation is strange here, free[0] should be on the same line as
> rte_mempool_put_bulk.
> 
> > +					free[0] = m;
> > +					blk_n = 1;
> > +				}
> > +			} else {
> > +				free[0] = m;
> > +				blk_n = 1;
> > +			}
> > +		}
> 
> This loop could be smaller, blk_n can only be equal to 0 in the first
> iteration, otherwise is >= 1.
> The first if statement can be merged with the second one: 
> 
> 	if (likely(m != NULL)) {
> 		if (likely(blk_n && m->pool == free[0]->pool)) {

This condition is a wrong also, it should be !blk_n || (m->pool ...

Why don't you keep a pointer to the mpool (e.g. m->pool == pool)?  It
seems to cost a little to deference two pointers to reach the pool's
one.

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH 2/5] net/mlx5: free buffers in bulk on Tx completion

[Yongseok Koh]

On Fri, Jun 30, 2017 at 02:43:21PM +0200, Nélio Laranjeiro wrote:
> On Fri, Jun 30, 2017 at 02:30:47PM +0200, Nélio Laranjeiro wrote:
> > On Wed, Jun 28, 2017 at 04:04:00PM -0700, Yongseok Koh wrote:
> > > When processing Tx completion, it is more efficient to free buffers in bulk
> > > using rte_mempool_put_bulk() if buffers are from a same mempool.
> > > 
> > > Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> > > ---
> > >  drivers/net/mlx5/mlx5_rxtx.c | 36 +++++++++++++++++++++++++++---------
> > >  1 file changed, 27 insertions(+), 9 deletions(-)
> > > 
> > > diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> > > index 43db06ad8..d81d630f7 100644
> > > --- a/drivers/net/mlx5/mlx5_rxtx.c
> > > +++ b/drivers/net/mlx5/mlx5_rxtx.c
> > > @@ -264,6 +264,8 @@ txq_complete(struct txq *txq)
> > >  	uint16_t cq_ci = txq->cq_ci;
> > >  	volatile struct mlx5_cqe *cqe = NULL;
> > >  	volatile struct mlx5_wqe_ctrl *ctrl;
> > > +	struct rte_mbuf *m, *free[elts_n];
> > > +	unsigned int blk_n = 0;
> > >  
> > >  	do {
> > >  		volatile struct mlx5_cqe *tmp;
> > > @@ -296,21 +298,37 @@ txq_complete(struct txq *txq)
> > >  	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
> > >  	/* Free buffers. */
> > >  	while (elts_free != elts_tail) {
> > > -		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
> > > -		struct rte_mbuf *elt_next =
> > > -			(*txq->elts)[(elts_free + 1) & elts_m];
> > > -
> > > +		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
> > > +		if (likely(m != NULL)) {
> > > +			if (blk_n) {
> > > +				if (likely(m->pool == free[0]->pool)) {
> > > +					free[blk_n++] = m;
> > > +				} else {
> > > +					rte_mempool_put_bulk(
> > > +							free[0]->pool,
> > > +							(void *)free,
> > > +							blk_n);
> > 
> > The indentation is strange here, free[0] should be on the same line as
> > rte_mempool_put_bulk.
> > 
> > > +					free[0] = m;
> > > +					blk_n = 1;
> > > +				}
> > > +			} else {
> > > +				free[0] = m;
> > > +				blk_n = 1;
> > > +			}
> > > +		}
> > 
> > This loop could be smaller, blk_n can only be equal to 0 in the first
> > iteration, otherwise is >= 1.
> > The first if statement can be merged with the second one: 
> > 
> > 	if (likely(m != NULL)) {
> > 		if (likely(blk_n && m->pool == free[0]->pool)) {
> 
> This condition is a wrong also, it should be !blk_n || (m->pool ...
> 
> Why don't you keep a pointer to the mpool (e.g. m->pool == pool)?  It
> seems to cost a little to deference two pointers to reach the pool's
> one.
Good point. The following will be the final streamlined code.
        /* Free buffers. */
        while (elts_free != elts_tail) {
                m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
                if (likely(m != NULL)) {
                        if (likely(m->pool == pool)) {
                                free[blk_n++] = m;
                        } else {
                                if (likely(pool != NULL))
                                        rte_mempool_put_bulk(pool,
                                                             (void *)free,
                                                             blk_n);
                                free[0] = m;
                                pool = m->pool;
                                blk_n = 1;
                        }
                }
        }
        if (blk_n)
                rte_mempool_put_bulk(pool, (void *)free, blk_n);


Thanks,
Yongseok

[dpdk-dev] [PATCH 3/5] net/mlx5: use buffer address for LKEY search

[Yongseok Koh]

When searching LKEY, if search key is mempool pointer, the 2nd cacheline
has to be accessed and it even requires to check whether a buffer is
indirect per every search. Instead, using address for search key can reduce
cycles taken. And caching the last hit entry is beneficial as well.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_mr.c   |  2 ++
 drivers/net/mlx5/mlx5_rxtx.c | 37 ++++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_rxtx.h |  3 +++
 3 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 0a3638460..f2484cb06 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -207,6 +207,8 @@ txq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx)
 			 sizeof(txq_ctrl->txq.mp2mr[0])));
 	}
 	/* Store the new entry. */
+	txq_ctrl->txq.mp2mr[idx].start = (uintptr_t)mr->addr;
+	txq_ctrl->txq.mp2mr[idx].end = (uintptr_t)mr->addr + mr->length;
 	txq_ctrl->txq.mp2mr[idx].mp = mp;
 	txq_ctrl->txq.mp2mr[idx].mr = mr;
 	txq_ctrl->txq.mp2mr[idx].lkey = htonl(mr->lkey);
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index d81d630f7..5cc38ccb7 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -77,7 +77,7 @@ static __rte_always_inline void
 txq_complete(struct txq *txq);
 
 static __rte_always_inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp);
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb);
 
 static __rte_always_inline void
 mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe);
@@ -355,7 +355,7 @@ txq_mb2mp(struct rte_mbuf *buf)
 }
 
 /**
- * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[].
+ * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
  * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
  * remove an entry first.
  *
@@ -368,27 +368,30 @@ txq_mb2mp(struct rte_mbuf *buf)
  *   mr->lkey on success, (uint32_t)-1 on failure.
  */
 static inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
 {
-	unsigned int i;
-	uint32_t lkey = (uint32_t)-1;
+	uint16_t i = txq->mr_cache_idx;
+	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
 
+	assert(i < RTE_DIM(txq->mp2mr));
+	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
+		return txq->mp2mr[i].lkey;
 	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
 		if (unlikely(txq->mp2mr[i].mp == NULL)) {
 			/* Unknown MP, add a new MR for it. */
 			break;
 		}
-		if (txq->mp2mr[i].mp == mp) {
+		if (txq->mp2mr[i].start <= addr &&
+		    txq->mp2mr[i].end >= addr) {
 			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
 			assert(htonl(txq->mp2mr[i].mr->lkey) ==
 			       txq->mp2mr[i].lkey);
-			lkey = txq->mp2mr[i].lkey;
-			break;
+			txq->mr_cache_idx = i;
+			return txq->mp2mr[i].lkey;
 		}
 	}
-	if (unlikely(lkey == (uint32_t)-1))
-		lkey = txq_mp2mr_reg(txq, mp, i);
-	return lkey;
+	txq->mr_cache_idx = 0;
+	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
 }
 
 /**
@@ -773,7 +776,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t){
 				htonl(length),
-				txq_mp2mr(txq, txq_mb2mp(buf)),
+				txq_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
@@ -812,7 +815,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		naddr = htonll(rte_pktmbuf_mtod(buf, uintptr_t));
 		*dseg = (rte_v128u32_t){
 			htonl(length),
-			txq_mp2mr(txq, txq_mb2mp(buf)),
+			txq_mb2mr(txq, buf),
 			naddr,
 			naddr >> 32,
 		};
@@ -1054,7 +1057,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
 				.byte_count = htonl(DATA_LEN(buf)),
-				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+				.lkey = txq_mb2mr(txq, buf),
 				.addr = htonll(addr),
 			};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1300,7 +1303,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+					.lkey = txq_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1607,7 +1610,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+					.lkey = txq_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1690,7 +1693,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t) {
 				htonl(length),
-				txq_mp2mr(txq, txq_mb2mp(buf)),
+				txq_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index f9b738b4e..a86f41abc 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -267,10 +267,13 @@ struct txq {
 	volatile uint32_t *cq_db; /* Completion queue doorbell. */
 	volatile void *bf_reg; /* Blueflame register. */
 	struct {
+		uintptr_t start; /* Start address of MR */
+		uintptr_t end;   /* End address of MR */
 		const struct rte_mempool *mp; /* Cached Memory Pool. */
 		struct ibv_mr *mr; /* Memory Region (for mp). */
 		uint32_t lkey; /* htonl(mr->lkey) */
 	} mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MP to MR translation table. */
+	uint16_t mr_cache_idx; /* Index of last hit entry. */
 	struct rte_mbuf *(*elts)[]; /* TX elements. */
 	struct mlx5_txq_stats stats; /* TX queue counters. */
 } __rte_cache_aligned;
-- 
2.11.0

Re: [dpdk-dev] [PATCH 3/5] net/mlx5: use buffer address for LKEY search

[Nélio Laranjeiro]

On Wed, Jun 28, 2017 at 04:04:01PM -0700, Yongseok Koh wrote:
> When searching LKEY, if search key is mempool pointer, the 2nd cacheline
> has to be accessed and it even requires to check whether a buffer is
> indirect per every search. Instead, using address for search key can reduce
> cycles taken. And caching the last hit entry is beneficial as well.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5_mr.c   |  2 ++
>  drivers/net/mlx5/mlx5_rxtx.c | 37 ++++++++++++++++++++-----------------
>  drivers/net/mlx5/mlx5_rxtx.h |  3 +++
>  3 files changed, 25 insertions(+), 17 deletions(-)
> 
>[...]
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index f9b738b4e..a86f41abc 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -267,10 +267,13 @@ struct txq {
>  	volatile uint32_t *cq_db; /* Completion queue doorbell. */
>  	volatile void *bf_reg; /* Blueflame register. */
>  	struct {
> +		uintptr_t start; /* Start address of MR */
> +		uintptr_t end;   /* End address of MR */

Extra space between the ';' and the comment.

>  		const struct rte_mempool *mp; /* Cached Memory Pool. */
>  		struct ibv_mr *mr; /* Memory Region (for mp). */
>  		uint32_t lkey; /* htonl(mr->lkey) */
>  	} mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MP to MR translation table. */
> +	uint16_t mr_cache_idx; /* Index of last hit entry. */
>  	struct rte_mbuf *(*elts)[]; /* TX elements. */
>  	struct mlx5_txq_stats stats; /* TX queue counters. */
>  } __rte_cache_aligned;
> -- 
> 2.11.0

The pointer to the mempool should be removed from this structure, it was
useful in the dataplane to retrieve the lkey, but with this new
implementation it becomes useless.
This also helps to keep the memory footprint of this array.  The control
plane can spend some cycles to retrieve the start/end addresses of the
mempool to compare them.

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH 3/5] net/mlx5: use buffer address for LKEY search

[Yongseok Koh]

On Fri, Jun 30, 2017 at 03:01:51PM +0200, Nélio Laranjeiro wrote:
> On Wed, Jun 28, 2017 at 04:04:01PM -0700, Yongseok Koh wrote:
> > When searching LKEY, if search key is mempool pointer, the 2nd cacheline
> > has to be accessed and it even requires to check whether a buffer is
> > indirect per every search. Instead, using address for search key can reduce
> > cycles taken. And caching the last hit entry is beneficial as well.
> > 
> > Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> > ---
> >  drivers/net/mlx5/mlx5_mr.c   |  2 ++
> >  drivers/net/mlx5/mlx5_rxtx.c | 37 ++++++++++++++++++++-----------------
> >  drivers/net/mlx5/mlx5_rxtx.h |  3 +++
> >  3 files changed, 25 insertions(+), 17 deletions(-)
> > 
> >[...]
> > diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> > index f9b738b4e..a86f41abc 100644
> > --- a/drivers/net/mlx5/mlx5_rxtx.h
> > +++ b/drivers/net/mlx5/mlx5_rxtx.h
> > @@ -267,10 +267,13 @@ struct txq {
> >  	volatile uint32_t *cq_db; /* Completion queue doorbell. */
> >  	volatile void *bf_reg; /* Blueflame register. */
> >  	struct {
> > +		uintptr_t start; /* Start address of MR */
> > +		uintptr_t end;   /* End address of MR */
> 
> Extra space between the ';' and the comment.
> 
> >  		const struct rte_mempool *mp; /* Cached Memory Pool. */
> >  		struct ibv_mr *mr; /* Memory Region (for mp). */
> >  		uint32_t lkey; /* htonl(mr->lkey) */
> >  	} mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MP to MR translation table. */
> > +	uint16_t mr_cache_idx; /* Index of last hit entry. */
> >  	struct rte_mbuf *(*elts)[]; /* TX elements. */
> >  	struct mlx5_txq_stats stats; /* TX queue counters. */
> >  } __rte_cache_aligned;
> > -- 
> > 2.11.0
> 
> The pointer to the mempool should be removed from this structure, it was
> useful in the dataplane to retrieve the lkey, but with this new
> implementation it becomes useless.
> This also helps to keep the memory footprint of this array.  The control
> plane can spend some cycles to retrieve the start/end addresses of the
> mempool to compare them.
Agree. Will make the change.

Thanks,
Yongseok

[dpdk-dev] [PATCH 4/5] net/mlx5: select Rx/Tx callbacks when starting device

[Yongseok Koh]

The callbacks are global to a device but the seletion is made every queue
configuration, which is redundant.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxq.c     | 2 --
 drivers/net/mlx5/mlx5_trigger.c | 3 +++
 drivers/net/mlx5/mlx5_txq.c     | 2 --
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 22be999e8..98b20eb92 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1226,8 +1226,6 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		DEBUG("%p: adding RX queue %p to list",
 		      (void *)dev, (void *)rxq_ctrl);
 		(*priv->rxqs)[idx] = &rxq_ctrl->rxq;
-		/* Update receive callback. */
-		priv_select_rx_function(priv);
 	}
 	priv_unlock(priv);
 	return -ret;
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 40f23da93..595a9e064 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -72,6 +72,9 @@ mlx5_dev_start(struct rte_eth_dev *dev)
 		priv_unlock(priv);
 		return 0;
 	}
+	/* Update Rx/Tx callback. */
+	priv_select_tx_function(priv);
+	priv_select_rx_function(priv);
 	DEBUG("%p: allocating and configuring hash RX queues", (void *)dev);
 	err = priv_create_hash_rxqs(priv);
 	if (!err)
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index f0729a2a8..c2d6c303d 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -497,8 +497,6 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		DEBUG("%p: adding TX queue %p to list",
 		      (void *)dev, (void *)txq_ctrl);
 		(*priv->txqs)[idx] = &txq_ctrl->txq;
-		/* Update send callback. */
-		priv_select_tx_function(priv);
 	}
 	priv_unlock(priv);
 	return -ret;
-- 
2.11.0

Re: [dpdk-dev] [PATCH 4/5] net/mlx5: select Rx/Tx callbacks when starting device

[Nélio Laranjeiro]

On Wed, Jun 28, 2017 at 04:04:02PM -0700, Yongseok Koh wrote:
> The callbacks are global to a device but the seletion is made every queue
> configuration, which is redundant.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
>[...]

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Yongseok Koh]

To make vectorized burst routines enabled, it is required to run on x86_64
architecture which can support at least SSE4.1. If all the conditions are
met, the vectorized burst functions are enabled automatically. The decision
is made individually on RX and TX. There's no PMD option to make a
selection.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/Makefile            |   10 +
 drivers/net/mlx5/mlx5_defs.h         |   18 +
 drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
 drivers/net/mlx5/mlx5_rxq.c          |   55 +-
 drivers/net/mlx5/mlx5_rxtx.c         |  342 ++------
 drivers/net/mlx5/mlx5_rxtx.h         |  286 ++++++-
 drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_txq.c          |    2 +-
 8 files changed, 1912 insertions(+), 280 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index daf80134b..7bb53b3c8 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -39,6 +39,11 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c
+ifneq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),)
+ifeq ($(CONFIG_RTE_ARCH_X86_64),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx_vec_sse.c
+endif
+endif
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_trigger.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mac.c
@@ -85,6 +90,11 @@ ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
 CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
 endif
 
+#check if flag for SSE4.1 is already on, if not set it up manually
+ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),)
+CFLAGS_mlx5_rxtx_vec_sse.o += -msse4.1
+endif
+
 include $(RTE_SDK)/mk/rte.lib.mk
 
 # Generate and clean-up mlx5_autoconf.h.
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 201bb3362..a76bc6f65 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -89,4 +89,22 @@
 /* Maximum Packet headers size (L2+L3+L4) for TSO. */
 #define MLX5_MAX_TSO_HEADER 128
 
+/* Default minimum number of Tx queues for vectorized Tx. */
+#define MLX5_VPMD_MIN_TXQS 4
+
+/* Threshold of buffer replenishment for vectorized Rx. */
+#define MLX5_VPMD_RXQ_RPLNSH_THRESH   64U
+
+/* Maximum size of burst for vectorized Rx. */
+#define MLX5_VPMD_RX_MAX_BURST        MLX5_VPMD_RXQ_RPLNSH_THRESH
+
+/*
+ * Maximum size of burst for vectorized Tx. This is related to the maximum size
+ * of Enhaned MPW (eMPW) WQE as vectorized Tx is supported with eMPW.
+ */
+#define MLX5_VPMD_TX_MAX_BURST        32U
+
+/* Number of packets vectorized Rx can simultaneously process in a loop. */
+#define MLX5_VPMD_DESCS_PER_LOOP      4
+
 #endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index eadf452b9..7f3cef547 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -723,7 +723,8 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 
 	};
 
-	if (dev->rx_pkt_burst == mlx5_rx_burst)
+	if (dev->rx_pkt_burst == mlx5_rx_burst ||
+	    dev->rx_pkt_burst == mlx5_rx_burst_vec)
 		return ptypes;
 	return NULL;
 }
@@ -924,7 +925,7 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	int ret = 0;
 	unsigned int i;
 	uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) =
-		mlx5_rx_burst;
+		dev->rx_pkt_burst;
 	unsigned int max_frame_len;
 	int rehash;
 	int restart = priv->started;
@@ -944,7 +945,7 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	/* Temporarily replace RX handler with a fake one, assuming it has not
 	 * been copied elsewhere. */
 	dev->rx_pkt_burst = removed_rx_burst;
-	/* Make sure everyone has left mlx5_rx_burst() and uses
+	/* Make sure everyone has left dev->rx_pkt_burst() and uses
 	 * removed_rx_burst() instead. */
 	rte_wmb();
 	usleep(1000);
@@ -1584,9 +1585,16 @@ priv_select_tx_function(struct priv *priv)
 	priv->dev->tx_pkt_burst = mlx5_tx_burst;
 	/* Select appropriate TX function. */
 	if (priv->mps == MLX5_MPW_ENHANCED) {
-		priv->dev->tx_pkt_burst =
-			mlx5_tx_burst_empw;
-		DEBUG("selected Enhanced MPW TX function");
+		if (priv_check_vec_tx_support(priv) > 0) {
+			if (priv_check_raw_vec_tx_support(priv) > 0)
+				priv->dev->tx_pkt_burst = mlx5_tx_burst_raw_vec;
+			else
+				priv->dev->tx_pkt_burst = mlx5_tx_burst_vec;
+			DEBUG("selected Enhanced MPW TX vectorized function");
+		} else {
+			priv->dev->tx_pkt_burst = mlx5_tx_burst_empw;
+			DEBUG("selected Enhanced MPW TX function");
+		}
 	} else if (priv->mps && priv->txq_inline) {
 		priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline;
 		DEBUG("selected MPW inline TX function");
@@ -1605,5 +1613,11 @@ priv_select_tx_function(struct priv *priv)
 void
 priv_select_rx_function(struct priv *priv)
 {
-	priv->dev->rx_pkt_burst = mlx5_rx_burst;
+	if (priv_check_vec_rx_support(priv) > 0) {
+		priv_prep_vec_rx_function(priv);
+		priv->dev->rx_pkt_burst = mlx5_rx_burst_vec;
+		DEBUG("selected RX vectorized function");
+	} else {
+		priv->dev->rx_pkt_burst = mlx5_rx_burst;
+	}
 }
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 98b20eb92..4842cda92 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -634,6 +634,41 @@ priv_rehash_flows(struct priv *priv)
 }
 
 /**
+ * Unlike regular Rx function, vPMD Rx doesn't replace mbufs immediately when
+ * receiving packets. Instead it replaces later in bulk. In rxq->elts[], entries
+ * from rq_pi to rq_ci are owned by device but the rest is already delivered to
+ * application. In order not to reuse those mbufs by rxq_alloc_elts(), this
+ * function must be called to replace used mbufs.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+static int
+rxq_trim_elts(struct rxq *rxq)
+{
+	const uint16_t q_n = (1 << rxq->elts_n);
+	const uint16_t q_mask = q_n - 1;
+	uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi);
+	uint16_t i;
+
+	if (!rxq->trim_elts)
+		return 0;
+	for (i = 0; i < used; ++i) {
+		struct rte_mbuf *buf;
+		buf = rte_pktmbuf_alloc(rxq->mp);
+		if (!buf)
+			return ENOMEM;
+		(*rxq->elts)[(rxq->rq_ci + i) & q_mask] = buf;
+	}
+	rxq->rq_pi = rxq->rq_ci;
+	rxq->trim_elts = 0;
+	return 0;
+}
+
+/**
  * Allocate RX queue elements.
  *
  * @param rxq_ctrl
@@ -800,6 +835,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl)
 		return err;
 	}
 	/* Snatch mbufs from original queue. */
+	claim_zero(rxq_trim_elts(&rxq_ctrl->rxq));
 	claim_zero(rxq_alloc_elts(rxq_ctrl, elts_n, rxq_ctrl->rxq.elts));
 	for (i = 0; i != elts_n; ++i) {
 		struct rte_mbuf *buf = (*rxq_ctrl->rxq.elts)[i];
@@ -860,6 +896,7 @@ rxq_setup(struct rxq_ctrl *tmpl)
 	tmpl->rxq.cqe_n = log2above(cq_info.cqe_cnt);
 	tmpl->rxq.cq_ci = 0;
 	tmpl->rxq.rq_ci = 0;
+	tmpl->rxq.rq_pi = 0;
 	tmpl->rxq.cq_db = cq_info.dbrec;
 	tmpl->rxq.wqes =
 		(volatile struct mlx5_wqe_data_seg (*)[])
@@ -993,7 +1030,12 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
 	if (priv->cqe_comp) {
 		attr.cq.comp_mask |= IBV_EXP_CQ_INIT_ATTR_FLAGS;
 		attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE;
-		cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
+		/*
+		 * For vectorized Rx, it must not be doubled in order to
+		 * make cq_ci and rq_ci aligned.
+		 */
+		if (rxq_check_vec_rx_support(&tmpl.rxq) < 0)
+			cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
 	}
 	tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, tmpl.channel, 0,
 				    &attr.cq);
@@ -1103,7 +1145,9 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
 	if (rxq_ctrl->rxq.elts_n) {
 		assert(1 << rxq_ctrl->rxq.elts_n == desc);
 		assert(rxq_ctrl->rxq.elts != tmpl.rxq.elts);
-		ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts);
+		ret = rxq_trim_elts(&rxq_ctrl->rxq);
+		if (!ret)
+			ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts);
 	} else
 		ret = rxq_alloc_elts(&tmpl, desc, NULL);
 	if (ret) {
@@ -1165,6 +1209,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	struct priv *priv = dev->data->dev_private;
 	struct rxq *rxq = (*priv->rxqs)[idx];
 	struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
+	const uint16_t desc_pad = MLX5_VPMD_DESCS_PER_LOOP; /* For vPMD. */
 	int ret;
 
 	if (mlx5_is_secondary())
@@ -1198,7 +1243,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		if (rxq_ctrl->rxq.elts_n != log2above(desc)) {
 			rxq_ctrl = rte_realloc(rxq_ctrl,
 					       sizeof(*rxq_ctrl) +
-					       desc * sizeof(struct rte_mbuf *),
+					       (desc + desc_pad) *
+						sizeof(struct rte_mbuf *),
 					       RTE_CACHE_LINE_SIZE);
 			if (!rxq_ctrl) {
 				ERROR("%p: unable to reallocate queue index %u",
@@ -1209,7 +1255,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		}
 	} else {
 		rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl) +
-					     desc * sizeof(struct rte_mbuf *),
+					     (desc + desc_pad) *
+					      sizeof(struct rte_mbuf *),
 					     0, socket);
 		if (rxq_ctrl == NULL) {
 			ERROR("%p: unable to allocate queue index %u",
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 5cc38ccb7..16bac5563 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -92,102 +92,6 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe,
 static __rte_always_inline uint32_t
 rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe);
 
-#ifndef NDEBUG
-
-/**
- * Verify or set magic value in CQE.
- *
- * @param cqe
- *   Pointer to CQE.
- *
- * @return
- *   0 the first time.
- */
-static inline int
-check_cqe_seen(volatile struct mlx5_cqe *cqe)
-{
-	static const uint8_t magic[] = "seen";
-	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
-	int ret = 1;
-	unsigned int i;
-
-	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
-		if (!ret || (*buf)[i] != magic[i]) {
-			ret = 0;
-			(*buf)[i] = magic[i];
-		}
-	return ret;
-}
-
-#endif /* NDEBUG */
-
-/**
- * Check whether CQE is valid.
- *
- * @param cqe
- *   Pointer to CQE.
- * @param cqes_n
- *   Size of completion queue.
- * @param ci
- *   Consumer index.
- *
- * @return
- *   0 on success, 1 on failure.
- */
-static inline int
-check_cqe(volatile struct mlx5_cqe *cqe,
-	  unsigned int cqes_n, const uint16_t ci)
-{
-	uint16_t idx = ci & cqes_n;
-	uint8_t op_own = cqe->op_own;
-	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
-	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
-
-	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
-		return 1; /* No CQE. */
-#ifndef NDEBUG
-	if ((op_code == MLX5_CQE_RESP_ERR) ||
-	    (op_code == MLX5_CQE_REQ_ERR)) {
-		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
-		uint8_t syndrome = err_cqe->syndrome;
-
-		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
-		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
-			return 0;
-		if (!check_cqe_seen(cqe))
-			ERROR("unexpected CQE error %u (0x%02x)"
-			      " syndrome 0x%02x",
-			      op_code, op_code, syndrome);
-		return 1;
-	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
-		   (op_code != MLX5_CQE_REQ)) {
-		if (!check_cqe_seen(cqe))
-			ERROR("unexpected CQE opcode %u (0x%02x)",
-			      op_code, op_code);
-		return 1;
-	}
-#endif /* NDEBUG */
-	return 0;
-}
-
-/**
- * Return the address of the WQE.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param  wqe_ci
- *   WQE consumer index.
- *
- * @return
- *   WQE address.
- */
-static inline uintptr_t *
-tx_mlx5_wqe(struct txq *txq, uint16_t ci)
-{
-	ci &= ((1 << txq->wqe_n) - 1);
-	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
-}
-
 /**
  * Return the size of tailroom of WQ.
  *
@@ -245,177 +149,6 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
 }
 
 /**
- * Manage TX completions.
- *
- * When sending a burst, mlx5_tx_burst() posts several WRs.
- *
- * @param txq
- *   Pointer to TX queue structure.
- */
-static inline void
-txq_complete(struct txq *txq)
-{
-	const uint16_t elts_n = 1 << txq->elts_n;
-	const uint16_t elts_m = elts_n - 1;
-	const unsigned int cqe_n = 1 << txq->cqe_n;
-	const unsigned int cqe_cnt = cqe_n - 1;
-	uint16_t elts_free = txq->elts_tail;
-	uint16_t elts_tail;
-	uint16_t cq_ci = txq->cq_ci;
-	volatile struct mlx5_cqe *cqe = NULL;
-	volatile struct mlx5_wqe_ctrl *ctrl;
-	struct rte_mbuf *m, *free[elts_n];
-	unsigned int blk_n = 0;
-
-	do {
-		volatile struct mlx5_cqe *tmp;
-
-		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
-		if (check_cqe(tmp, cqe_n, cq_ci))
-			break;
-		cqe = tmp;
-#ifndef NDEBUG
-		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
-			if (!check_cqe_seen(cqe))
-				ERROR("unexpected compressed CQE, TX stopped");
-			return;
-		}
-		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
-		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
-			if (!check_cqe_seen(cqe))
-				ERROR("unexpected error CQE, TX stopped");
-			return;
-		}
-#endif /* NDEBUG */
-		++cq_ci;
-	} while (1);
-	if (unlikely(cqe == NULL))
-		return;
-	txq->wqe_pi = ntohs(cqe->wqe_counter);
-	ctrl = (volatile struct mlx5_wqe_ctrl *)
-		tx_mlx5_wqe(txq, txq->wqe_pi);
-	elts_tail = ctrl->ctrl3;
-	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
-	/* Free buffers. */
-	while (elts_free != elts_tail) {
-		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
-		if (likely(m != NULL)) {
-			if (blk_n) {
-				if (likely(m->pool == free[0]->pool)) {
-					free[blk_n++] = m;
-				} else {
-					rte_mempool_put_bulk(
-							free[0]->pool,
-							(void *)free,
-							blk_n);
-					free[0] = m;
-					blk_n = 1;
-				}
-			} else {
-				free[0] = m;
-				blk_n = 1;
-			}
-		}
-	}
-	if (blk_n)
-		rte_mempool_put_bulk(free[0]->pool, (void *)free, blk_n);
-#ifndef NDEBUG
-	elts_free = txq->elts_tail;
-	/* Poisoning. */
-	while (elts_free != elts_tail) {
-		memset(&(*txq->elts)[elts_free & elts_m],
-		       0x66,
-		       sizeof((*txq->elts)[elts_free & elts_m]));
-		++elts_free;
-	}
-#endif
-	txq->cq_ci = cq_ci;
-	txq->elts_tail = elts_tail;
-	/* Update the consumer index. */
-	rte_wmb();
-	*txq->cq_db = htonl(cq_ci);
-}
-
-/**
- * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
- * the cloned mbuf is allocated is returned instead.
- *
- * @param buf
- *   Pointer to mbuf.
- *
- * @return
- *   Memory pool where data is located for given mbuf.
- */
-static struct rte_mempool *
-txq_mb2mp(struct rte_mbuf *buf)
-{
-	if (unlikely(RTE_MBUF_INDIRECT(buf)))
-		return rte_mbuf_from_indirect(buf)->pool;
-	return buf->pool;
-}
-
-/**
- * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
- * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
- * remove an entry first.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param[in] mp
- *   Memory Pool for which a Memory Region lkey must be returned.
- *
- * @return
- *   mr->lkey on success, (uint32_t)-1 on failure.
- */
-static inline uint32_t
-txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
-{
-	uint16_t i = txq->mr_cache_idx;
-	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
-
-	assert(i < RTE_DIM(txq->mp2mr));
-	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
-		return txq->mp2mr[i].lkey;
-	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
-		if (unlikely(txq->mp2mr[i].mp == NULL)) {
-			/* Unknown MP, add a new MR for it. */
-			break;
-		}
-		if (txq->mp2mr[i].start <= addr &&
-		    txq->mp2mr[i].end >= addr) {
-			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
-			assert(htonl(txq->mp2mr[i].mr->lkey) ==
-			       txq->mp2mr[i].lkey);
-			txq->mr_cache_idx = i;
-			return txq->mp2mr[i].lkey;
-		}
-	}
-	txq->mr_cache_idx = 0;
-	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
-}
-
-/**
- * Ring TX queue doorbell.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param wqe
- *   Pointer to the last WQE posted in the NIC.
- */
-static inline void
-mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
-{
-	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
-	volatile uint64_t *src = ((volatile uint64_t *)wqe);
-
-	rte_wmb();
-	*txq->qp_db = htonl(txq->wqe_ci);
-	/* Ensure ordering between DB record and BF copy. */
-	rte_wmb();
-	*dst = *src;
-}
-
-/**
  * DPDK callback to check the status of a tx descriptor.
  *
  * @param tx_queue
@@ -2141,3 +1874,78 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	(void)pkts_n;
 	return 0;
 }
+
+/*
+ * Vectorized Rx/Tx routines are not compiled in when required vector
+ * instructions are not supported on a target architecture. The following null
+ * stubs are needed for linkage when those are not included outside of this file
+ * (e.g.  mlx5_rxtx_vec_sse.c for x86).
+ */
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_txq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_txq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_rxq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+int __attribute__((weak))
+txq_check_vec_tx_support(struct txq *txq)
+{
+	(void)txq;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_raw_vec_tx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_vec_tx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+rxq_check_vec_rx_support(struct rxq *rxq)
+{
+	(void)rxq;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_vec_rx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+void __attribute__((weak))
+priv_prep_vec_rx_function(struct priv *priv)
+{
+	(void)priv;
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index a86f41abc..200bd7924 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -115,10 +115,13 @@ struct rxq {
 	unsigned int port_id:8;
 	unsigned int rss_hash:1; /* RSS hash result is enabled. */
 	unsigned int mark:1; /* Marked flow available on the queue. */
-	unsigned int :8; /* Remaining bits. */
+	unsigned int pending_err:1; /* CQE error needs to be handled. */
+	unsigned int trim_elts:1; /* Whether elts needs clean-up. */
+	unsigned int :6; /* Remaining bits. */
 	volatile uint32_t *rq_db;
 	volatile uint32_t *cq_db;
 	uint16_t rq_ci;
+	uint16_t rq_pi;
 	uint16_t cq_ci;
 	volatile struct mlx5_wqe_data_seg(*wqes)[];
 	volatile struct mlx5_cqe(*cqes)[];
@@ -126,6 +129,8 @@ struct rxq {
 	struct rte_mbuf *(*elts)[];
 	struct rte_mempool *mp;
 	struct mlx5_rxq_stats stats;
+	uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */
+	struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
 } __rte_cache_aligned;
 
 /* RX queue control descriptor. */
@@ -261,6 +266,7 @@ struct txq {
 	uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */
 	uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
 	uint32_t qp_num_8s; /* QP number shifted by 8. */
+	uint32_t flags; /* Flags for Tx Queue. */
 	volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
 	volatile void *wqes; /* Work queue (use volatile to write into). */
 	volatile uint32_t *qp_db; /* Work queue doorbell. */
@@ -339,10 +345,288 @@ uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t);
 int mlx5_rx_descriptor_status(void *, uint16_t);
 int mlx5_tx_descriptor_status(void *, uint16_t);
 
+/* Vectorized version of mlx5_rxtx.c */
+int txq_check_vec_tx_support(struct txq *);
+int priv_check_raw_vec_tx_support(struct priv *);
+int priv_check_vec_tx_support(struct priv *);
+int rxq_check_vec_rx_support(struct rxq *);
+int priv_check_vec_rx_support(struct priv *);
+void priv_prep_vec_rx_function(struct priv *);
+uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t);
+uint16_t mlx5_tx_burst_vec(void *, struct rte_mbuf **, uint16_t);
+uint16_t mlx5_rx_burst_vec(void *, struct rte_mbuf **, uint16_t);
+
 /* mlx5_mr.c */
 
 struct ibv_mr *mlx5_mp2mr(struct ibv_pd *, struct rte_mempool *);
 void txq_mp2mr_iter(struct rte_mempool *, void *);
 uint32_t txq_mp2mr_reg(struct txq *, struct rte_mempool *, unsigned int);
 
+#ifndef NDEBUG
+
+/**
+ * Verify or set magic value in CQE.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ *
+ * @return
+ *   0 the first time.
+ */
+static inline int
+check_cqe_seen(volatile struct mlx5_cqe *cqe)
+{
+	static const uint8_t magic[] = "seen";
+	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
+	int ret = 1;
+	unsigned int i;
+
+	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
+		if (!ret || (*buf)[i] != magic[i]) {
+			ret = 0;
+			(*buf)[i] = magic[i];
+		}
+	return ret;
+}
+
+#endif /* NDEBUG */
+
+/**
+ * Check whether CQE is valid.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ * @param cqes_n
+ *   Size of completion queue.
+ * @param ci
+ *   Consumer index.
+ *
+ * @return
+ *   0 on success, 1 on failure.
+ */
+static inline int
+check_cqe(volatile struct mlx5_cqe *cqe,
+	  unsigned int cqes_n, const uint16_t ci)
+{
+	uint16_t idx = ci & cqes_n;
+	uint8_t op_own = cqe->op_own;
+	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
+	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
+
+	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
+		return 1; /* No CQE. */
+#ifndef NDEBUG
+	if ((op_code == MLX5_CQE_RESP_ERR) ||
+	    (op_code == MLX5_CQE_REQ_ERR)) {
+		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
+		uint8_t syndrome = err_cqe->syndrome;
+
+		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
+		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
+			return 0;
+		if (!check_cqe_seen(cqe))
+			ERROR("unexpected CQE error %u (0x%02x)"
+			      " syndrome 0x%02x",
+			      op_code, op_code, syndrome);
+		return 1;
+	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
+		   (op_code != MLX5_CQE_REQ)) {
+		if (!check_cqe_seen(cqe))
+			ERROR("unexpected CQE opcode %u (0x%02x)",
+			      op_code, op_code);
+		return 1;
+	}
+#endif /* NDEBUG */
+	return 0;
+}
+
+/**
+ * Return the address of the WQE.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param  wqe_ci
+ *   WQE consumer index.
+ *
+ * @return
+ *   WQE address.
+ */
+static inline uintptr_t *
+tx_mlx5_wqe(struct txq *txq, uint16_t ci)
+{
+	ci &= ((1 << txq->wqe_n) - 1);
+	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
+}
+
+/**
+ * Manage TX completions.
+ *
+ * When sending a burst, mlx5_tx_burst() posts several WRs.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ */
+static inline void
+txq_complete(struct txq *txq)
+{
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int cqe_n = 1 << txq->cqe_n;
+	const unsigned int cqe_cnt = cqe_n - 1;
+	uint16_t elts_free = txq->elts_tail;
+	uint16_t elts_tail;
+	uint16_t cq_ci = txq->cq_ci;
+	volatile struct mlx5_cqe *cqe = NULL;
+	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	unsigned int blk_n = 0;
+
+	do {
+		volatile struct mlx5_cqe *tmp;
+
+		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
+		if (check_cqe(tmp, cqe_n, cq_ci))
+			break;
+		cqe = tmp;
+#ifndef NDEBUG
+		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
+			if (!check_cqe_seen(cqe))
+				ERROR("unexpected compressed CQE, TX stopped");
+			return;
+		}
+		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
+		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
+			if (!check_cqe_seen(cqe))
+				ERROR("unexpected error CQE, TX stopped");
+			return;
+		}
+#endif /* NDEBUG */
+		++cq_ci;
+	} while (1);
+	if (unlikely(cqe == NULL))
+		return;
+	txq->wqe_pi = ntohs(cqe->wqe_counter);
+	ctrl = (volatile struct mlx5_wqe_ctrl *)
+		tx_mlx5_wqe(txq, txq->wqe_pi);
+	elts_tail = ctrl->ctrl3;
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
+	/* Free buffers. */
+	while (elts_free != elts_tail) {
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (blk_n) {
+				if (likely(m->pool == free[0]->pool)) {
+					free[blk_n++] = m;
+				} else {
+					rte_mempool_put_bulk(
+							free[0]->pool,
+							(void *)free,
+							blk_n);
+					free[0] = m;
+					blk_n = 1;
+				}
+			} else {
+				free[0] = m;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(free[0]->pool, (void *)free, blk_n);
+#ifndef NDEBUG
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
+		memset(&(*txq->elts)[elts_free & elts_m],
+		       0x66,
+		       sizeof((*txq->elts)[elts_free & elts_m]));
+		++elts_free;
+	}
+#endif
+	txq->cq_ci = cq_ci;
+	txq->elts_tail = elts_tail;
+	/* Update the consumer index. */
+	rte_wmb();
+	*txq->cq_db = htonl(cq_ci);
+}
+
+/**
+ * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
+ * the cloned mbuf is allocated is returned instead.
+ *
+ * @param buf
+ *   Pointer to mbuf.
+ *
+ * @return
+ *   Memory pool where data is located for given mbuf.
+ */
+static struct rte_mempool *
+txq_mb2mp(struct rte_mbuf *buf)
+{
+	if (unlikely(RTE_MBUF_INDIRECT(buf)))
+		return rte_mbuf_from_indirect(buf)->pool;
+	return buf->pool;
+}
+
+/**
+ * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
+ * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
+ * remove an entry first.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param[in] mp
+ *   Memory Pool for which a Memory Region lkey must be returned.
+ *
+ * @return
+ *   mr->lkey on success, (uint32_t)-1 on failure.
+ */
+static inline uint32_t
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
+{
+	uint16_t i = txq->mr_cache_idx;
+	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
+
+	assert(i < RTE_DIM(txq->mp2mr));
+	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
+		return txq->mp2mr[i].lkey;
+	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
+		if (unlikely(txq->mp2mr[i].mp == NULL)) {
+			/* Unknown MP, add a new MR for it. */
+			break;
+		}
+		if (txq->mp2mr[i].start <= addr &&
+		    txq->mp2mr[i].end >= addr) {
+			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
+			assert(htonl(txq->mp2mr[i].mr->lkey) ==
+			       txq->mp2mr[i].lkey);
+			txq->mr_cache_idx = i;
+			return txq->mp2mr[i].lkey;
+		}
+	}
+	txq->mr_cache_idx = 0;
+	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
+}
+
+/**
+ * Ring TX queue doorbell.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param wqe
+ *   Pointer to the last WQE posted in the NIC.
+ */
+static inline void
+mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
+{
+	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
+	volatile uint64_t *src = ((volatile uint64_t *)wqe);
+
+	rte_wmb();
+	*txq->qp_db = htonl(txq->wqe_ci);
+	/* Ensure ordering between DB record and BF copy. */
+	rte_wmb();
+	*dst = *src;
+}
+
 #endif /* RTE_PMD_MLX5_RXTX_H_ */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.c b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
new file mode 100644
index 000000000..566a60635
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
@@ -0,0 +1,1451 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5_hw.h>
+#include <infiniband/arch.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+/* DPDK headers don't like -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+#include <smmintrin.h>
+
+#ifndef __INTEL_COMPILER
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#endif
+
+/**
+ * Fill in DSEGs in Tx descriptors (WQE).
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param dseg
+ *   Pointer to DSEG to be writen.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param n
+ *   Number of packets to be filled.
+ */
+static inline void
+txq_wr_dseg_v(struct txq *txq, __m128i *dseg,
+	      struct rte_mbuf **pkts, unsigned int n)
+{
+	unsigned int pos;
+	uintptr_t addr;
+	const __m128i shuf_mask_dseg =
+		_mm_set_epi8(8,  9, 10, 11, /* addr, bswap64 */
+			    12, 13, 14, 15,
+			     7,  6,  5,  4, /* lkey */
+			     0,  1,  2,  3  /* length, bswap32 */);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t tx_byte = 0;
+#endif
+
+	for (pos = 0; pos < n; ++pos, ++dseg) {
+		__m128i desc;
+		struct rte_mbuf *pkt = pkts[pos];
+
+		addr = rte_pktmbuf_mtod(pkt, uintptr_t);
+		desc = _mm_set_epi32(addr >> 32,
+				     addr,
+				     txq_mb2mr(txq, pkt),
+				     DATA_LEN(pkt));
+		desc = _mm_shuffle_epi8(desc, shuf_mask_dseg);
+		_mm_store_si128(dseg, desc);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		tx_byte += DATA_LEN(pkt);
+#endif
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.obytes += tx_byte;
+#endif
+}
+
+/**
+ * Count the number of continuous single segment packets. The first packet must
+ * be a single segment packet.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ *
+ * @return
+ *   Number of continuous single segment packets.
+ */
+static inline unsigned int
+txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	unsigned int pos;
+
+	if (!pkts_n)
+		return 0;
+	assert(NB_SEGS(pkts[0]) == 1);
+	/* Count the number of continuous single segment packets. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if (NB_SEGS(pkts[pos]) > 1)
+			break;
+	return pos;
+}
+
+/**
+ * Count the number of packets having same ol_flags and calculate cs_flags.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ * @param cs_flags
+ *   Pointer of flags to be returned.
+ *
+ * @return
+ *   Number of packets having same ol_flags.
+ */
+static inline unsigned int
+txq_calc_offload(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+		 uint8_t *cs_flags)
+{
+	unsigned int pos;
+	const uint64_t ol_mask =
+		PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
+		PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
+		PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
+
+	if (!pkts_n)
+		return 0;
+	/* Count the number of packets having same ol_flags. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
+			break;
+	/* Should open another MPW session for the rest. */
+	if (pkts[0]->ol_flags &
+	    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+		const uint64_t is_tunneled =
+			pkts[0]->ol_flags &
+			(PKT_TX_TUNNEL_GRE |
+			 PKT_TX_TUNNEL_VXLAN);
+
+		if (is_tunneled && txq->tunnel_en) {
+			*cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
+				    MLX5_ETH_WQE_L4_INNER_CSUM;
+			if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+				*cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+		} else {
+			*cs_flags = MLX5_ETH_WQE_L3_CSUM |
+				    MLX5_ETH_WQE_L4_CSUM;
+		}
+	}
+	return pos;
+}
+
+/**
+ * Send multi-segmented packets until it encounters a single segment packet in
+ * the pkts list.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static uint16_t
+txq_scatter_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n;
+	volatile struct mlx5_wqe *wqe = NULL;
+
+	assert(elts_n > pkts_n);
+	txq_complete(txq);
+	/* A CQE slot must always be available. */
+	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
+	if (unlikely(!pkts_n))
+		return 0;
+	for (n = 0; n < pkts_n; ++n) {
+		struct rte_mbuf *buf = pkts[n];
+		unsigned int segs_n = buf->nb_segs;
+		unsigned int ds = nb_dword_in_hdr;
+		unsigned int len = PKT_LEN(buf);
+		uint16_t wqe_ci = txq->wqe_ci;
+		const __m128i shuf_mask_ctrl =
+			_mm_set_epi8(15, 14, 13, 12,
+				      8,  9, 10, 11, /* bswap32 */
+				      4,  5,  6,  7, /* bswap32 */
+				      0,  1,  2,  3  /* bswap32 */);
+		uint8_t cs_flags = 0;
+		uint16_t max_elts;
+		uint16_t max_wqe;
+		__m128i *t_wqe, *dseg;
+		__m128i ctrl;
+
+		assert(segs_n);
+		max_elts = elts_n - (elts_head - txq->elts_tail);
+		max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi);
+		/*
+		 * A MPW session consumes 2 WQEs at most to
+		 * include MLX5_MPW_DSEG_MAX pointers.
+		 */
+		if (segs_n == 1 ||
+		    max_elts < segs_n || max_wqe < 2)
+			break;
+		wqe = &((volatile struct mlx5_wqe64 *)
+			 txq->wqes)[wqe_ci & wq_mask].hdr;
+		if (buf->ol_flags &
+		     (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+			const uint64_t is_tunneled = buf->ol_flags &
+						      (PKT_TX_TUNNEL_GRE |
+						       PKT_TX_TUNNEL_VXLAN);
+
+			if (is_tunneled && txq->tunnel_en) {
+				cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
+					   MLX5_ETH_WQE_L4_INNER_CSUM;
+				if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+					cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+			} else {
+				cs_flags = MLX5_ETH_WQE_L3_CSUM |
+					   MLX5_ETH_WQE_L4_CSUM;
+			}
+		}
+		/* Title WQEBB pointer. */
+		t_wqe = (__m128i *)wqe;
+		dseg = (__m128i *)(wqe + 1);
+		do {
+			if (!(ds++ % nb_dword_per_wqebb)) {
+				dseg = (__m128i *)
+					&((volatile struct mlx5_wqe64 *)
+					   txq->wqes)[++wqe_ci & wq_mask];
+			}
+			txq_wr_dseg_v(txq, dseg++, &buf, 1);
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			buf = buf->next;
+		} while (--segs_n);
+		if (ds % nb_dword_per_wqebb)
+			++wqe_ci;
+		/* Fill CTRL in the header. */
+		ctrl = _mm_set_epi32(0, 0, txq->qp_num_8s | ds,
+				     MLX5_OPC_MOD_MPW << 24 |
+				     txq->wqe_ci << 8 | MLX5_OPCODE_TSO);
+		ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+		_mm_store_si128(t_wqe, ctrl);
+		/* Fill ESEG in the header. */
+		_mm_store_si128(t_wqe + 1,
+				_mm_set_epi16(0, 0, 0, 0,
+					      htons(len), cs_flags,
+					      0, 0));
+		txq->wqe_ci = wqe_ci;
+	}
+	if (!n)
+		return 0;
+	txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
+	txq->elts_head = elts_head;
+	if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+		wqe->ctrl[2] = htonl(8);
+		wqe->ctrl[3] = txq->elts_head;
+		txq->elts_comp = 0;
+		++txq->cq_pi;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += n;
+#endif
+	mlx5_tx_dbrec(txq, wqe);
+	return n;
+}
+
+/**
+ * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
+ * it returns to make it processed by txq_scatter_v(). All the packets in
+ * the pkts list should be single segment packets having same offload flags.
+ * This must be checked by txq_check_multiseg() and txq_calc_offload().
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent.
+ * @param cs_flags
+ *   Checksum offload flags to be written in the descriptor.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static inline uint16_t
+txq_burst_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+	    uint8_t cs_flags)
+{
+	struct rte_mbuf **elts;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n = 0;
+	unsigned int pos;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	uint32_t comp_req = 0;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	uint16_t wq_idx = txq->wqe_ci & wq_mask;
+	volatile struct mlx5_wqe64 *wq =
+		&((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx];
+	volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq;
+	const __m128i shuf_mask_ctrl =
+		_mm_set_epi8(15, 14, 13, 12,
+			      8,  9, 10, 11, /* bswap32 */
+			      4,  5,  6,  7, /* bswap32 */
+			      0,  1,  2,  3  /* bswap32 */);
+	__m128i *t_wqe, *dseg;
+	__m128i ctrl;
+
+	/* Make sure all packets can fit into a single WQE. */
+	assert(pkts_n <= MLX5_VPMD_TX_MAX_BURST);
+	assert(elts_n > pkts_n);
+	txq_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	/* A CQE slot must always be available. */
+	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
+	if (unlikely(!pkts_n))
+		return 0;
+	elts = &(*txq->elts)[elts_head & elts_m];
+	/* Loop for available tailroom first. */
+	n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n);
+	for (pos = 0; pos < (n & -2); pos += 2)
+		_mm_storeu_si128((__m128i *)&elts[pos],
+				 _mm_loadu_si128((__m128i *)&pkts[pos]));
+	if (n & 1)
+		elts[pos] = pkts[pos];
+	/* Check if it crosses the end of the queue. */
+	if (unlikely(n < pkts_n)) {
+		elts = &(*txq->elts)[0];
+		for (pos = 0; pos < pkts_n - n; ++pos)
+			elts[pos] = pkts[n + pos];
+	}
+	txq->elts_head += pkts_n;
+	/* Save title WQEBB pointer. */
+	t_wqe = (__m128i *)wqe;
+	dseg = (__m128i *)(wqe + 1);
+	/* Calculate the number of entries to the end. */
+	n = RTE_MIN(
+		(wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr,
+		pkts_n);
+	/* Fill DSEGs. */
+	txq_wr_dseg_v(txq, dseg, pkts, n);
+	/* Check if it crosses the end of the queue. */
+	if (n < pkts_n) {
+		dseg = (__m128i *)txq->wqes;
+		txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n);
+	}
+	if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
+		txq->elts_comp += pkts_n;
+	} else {
+		/* Request a completion. */
+		txq->elts_comp = 0;
+		++txq->cq_pi;
+		comp_req = 8;
+	}
+	/* Fill CTRL in the header. */
+	ctrl = _mm_set_epi32(txq->elts_head, comp_req,
+			     txq->qp_num_8s | (pkts_n + 2),
+			     MLX5_OPC_MOD_ENHANCED_MPSW << 24 |
+				txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW);
+	ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+	_mm_store_si128(t_wqe, ctrl);
+	/* Fill ESEG in the header. */
+	_mm_store_si128(t_wqe + 1,
+			_mm_set_epi8(0, 0, 0, 0,
+				     0, 0, 0, 0,
+				     0, 0, 0, cs_flags,
+				     0, 0, 0, 0));
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += pkts_n;
+#endif
+	txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) /
+		       nb_dword_per_wqebb;
+	/* Ring QP doorbell. */
+	mlx5_tx_dbrec(txq, wqe);
+	return pkts_n;
+}
+
+/**
+ * DPDK callback for vectorized TX.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
+		      uint16_t pkts_n)
+{
+	struct txq *txq = (struct txq *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint16_t n;
+		uint16_t ret;
+
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, 0);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * DPDK callback for vectorized TX with multi-seg packets and offload.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct txq *txq = (struct txq *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint8_t cs_flags = 0;
+		uint16_t n;
+		uint16_t ret;
+
+		/* Transmit multi-seg packets in the head of pkts list. */
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) &&
+		    NB_SEGS(pkts[nb_tx]) > 1)
+			nb_tx += txq_scatter_v(
+					txq, &pkts[nb_tx], pkts_n - nb_tx);
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS))
+			n = txq_check_multiseg(&pkts[nb_tx], n);
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
+			n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * Store free buffers to RX SW ring.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be stored.
+ * @param pkts_n
+ *   Number of packets to be stored.
+ */
+static inline void
+rxq_copy_mbuf_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t n)
+{
+	const uint16_t q_mask = (1 << rxq->elts_n) - 1;
+	struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask];
+	unsigned int pos;
+	uint16_t p = n & -2;
+
+	for (pos = 0; pos < p; pos += 2) {
+		__m128i mbp;
+
+		mbp = _mm_loadu_si128((__m128i *)&elts[pos]);
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp);
+	}
+	if (n & 1)
+		pkts[pos] = elts[pos];
+}
+
+/**
+ * Replenish buffers for RX in bulk.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ */
+static inline void
+rxq_replenish_bulk_mbuf(struct rxq *rxq)
+{
+	const uint16_t q_n = 1 << rxq->elts_n;
+	const uint16_t q_mask = q_n - 1;
+	const uint16_t elts_idx = rxq->rq_ci & q_mask;
+	struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
+	volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx];
+	uint16_t n = q_n - (rxq->rq_ci - rxq->rq_pi);
+	unsigned int i;
+
+	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH);
+	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
+	if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
+		rxq->stats.rx_nombuf += n;
+		return;
+	}
+	for (i = 0; i < n; ++i)
+		wq[i].addr = htonll(rte_pktmbuf_mtod(elts[i], uintptr_t));
+	rxq->rq_ci += n;
+	rte_wmb();
+	*rxq->rq_db = htonl(rxq->rq_ci);
+}
+
+/**
+ * Decompress a compressed CQE and fill in mbufs in RX SW ring with data
+ * extracted from the title CQE.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cq
+ *   Pointer to CQ array having a compressed CQE at first.
+ * @param elts
+ *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
+ *   the title CQE to be copied to the rest of mbufs.
+ */
+static inline void
+rxq_cq_decompress_v(struct rxq *rxq,
+		    volatile struct mlx5_cqe *cq,
+		    struct rte_mbuf **elts)
+{
+	volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + 1);
+	struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */
+	unsigned int pos;
+	unsigned int i;
+	unsigned int inv = 0;
+	/* Mask to shuffle from extracted mini CQE to mbuf. */
+	const __m128i shuf_mask1 =
+		_mm_set_epi8(0,  1,  2,  3, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			     6,  7,         /* data_len, bswap16 */
+			    -1, -1,  6,  7, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	const __m128i shuf_mask2 =
+		_mm_set_epi8(8,  9, 10, 11, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			    14, 15,         /* data_len, bswap16 */
+			    -1, -1, 14, 15, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	/* Restore the compressed count. Must be 16 bits. */
+	const uint16_t mcqe_n = t_pkt->data_len +
+				(rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i rearm =
+		_mm_loadu_si128((__m128i *)&t_pkt->rearm_data);
+	const __m128i rxdf =
+		_mm_loadu_si128((__m128i *)&t_pkt->rx_descriptor_fields1);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0, 0);
+	const uint32_t flow_tag = t_pkt->hash.fdir.hi;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     14, 15,  6,  7,
+			     10, 11,  2,  3);
+#endif
+
+	/* Compile time sanity check for this function. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+	/*
+	 * A. load mCQEs into a 128bit register.
+	 * B. store rearm data to mbuf.
+	 * C. combine data from mCQEs with rx_descriptor_fields1.
+	 * D. store rx_descriptor_fields1.
+	 * E. store flow tag (rte_flow mark).
+	 */
+	for (pos = 0; pos < mcqe_n; ) {
+		__m128i mcqe1, mcqe2;
+		__m128i rxdf1, rxdf2;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt, invalid_mask;
+#endif
+
+		if (!(pos & 0x7) && pos + 8 < mcqe_n)
+			rte_prefetch0((void *)(cq + pos + 8));
+		/* A.1 load mCQEs into a 128bit register. */
+		mcqe1 = _mm_loadu_si128((__m128i *)&mcq[pos % 8]);
+		mcqe2 = _mm_loadu_si128((__m128i *)&mcq[pos % 8 + 2]);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 1]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe1, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe1, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 1]->rx_descriptor_fields1,
+				 rxdf2);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos + 2]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 3]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe2, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe2, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 2]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 3]->rx_descriptor_fields1,
+				 rxdf2);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		invalid_mask = ones << ((mcqe_n - pos) * sizeof(uint16_t) * 8);
+		mcqe1 = _mm_srli_si128(mcqe1, 4);
+		byte_cnt = _mm_blend_epi16(mcqe1, mcqe2, 0xcc);
+		byte_cnt = _mm_shuffle_epi8(byte_cnt, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		if (rxq->mark) {
+			/* E.1 store flow tag (rte_flow mark). */
+			elts[pos]->hash.fdir.hi = flow_tag;
+			elts[pos + 1]->hash.fdir.hi = flow_tag;
+			elts[pos + 2]->hash.fdir.hi = flow_tag;
+			elts[pos + 3]->hash.fdir.hi = flow_tag;
+		}
+		pos += MLX5_VPMD_DESCS_PER_LOOP;
+		/* Move to next CQE and invalidate consumed CQEs. */
+		if (!(pos & 0x7) && pos < mcqe_n) {
+			mcq = (void *)(cq + pos);
+			for (i = 0; i < 8; ++i)
+				cq[inv++].op_own = MLX5_CQE_INVALIDATE;
+		}
+	}
+	/* Invalidate the rest of CQEs. */
+	for (; inv < mcqe_n; ++inv)
+		cq[inv].op_own = MLX5_CQE_INVALIDATE;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += mcqe_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	rxq->cq_ci += mcqe_n;
+}
+
+/*
+ * The index to the array should have:
+ * bit[1:0] = l3_hdr_type, bit[2] = tunneled, bit[3] = outer_l3_type
+ */
+static uint32_t mlx5_ptype_table[] = {
+	RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b0001 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b0010 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b0101 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b0110 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b1001 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b1010 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b1101 */
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b1110 */
+	RTE_PTYPE_ALL_MASK			     /* b1111 */
+};
+
+/**
+ * Calculate packet type and offload flag for mbuf and store it.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cqes[4]
+ *   Array of four 16bytes CQEs extracted from the original CQE.
+ * @param op_err
+ *   Opcode vector having responder error status. Each field is 4B.
+ * @param pkts
+ *   Pointer to array of packets to be filled.
+ */
+static inline void
+rxq_cq_to_ptype_oflags_v(struct rxq *rxq, __m128i cqes[4], __m128i op_err,
+			 struct rte_mbuf **pkts)
+{
+	__m128i pinfo0;
+	__m128i pinfo1;
+	__m128i pinfo, ptype;
+	__m128i ol_flags = _mm_set1_epi32(
+			rxq->rss_hash * PKT_RX_RSS_HASH);
+	__m128i cv_flags;
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ptype_mask =
+		_mm_set_epi32(0xd06, 0xd06, 0xd06, 0xd06);
+	const __m128i ptype_ol_mask =
+		_mm_set_epi32(0x106, 0x106, 0x106, 0x106);
+	const __m128i pinfo_mask =
+		_mm_set_epi32(0x3, 0x3, 0x3, 0x3);
+	const __m128i cv_flag_sel =
+		_mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0,
+			     (uint8_t)((PKT_RX_IP_CKSUM_GOOD |
+					     PKT_RX_L4_CKSUM_GOOD) >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1),
+			     (uint8_t)(PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED),
+			     0);
+	const __m128i cv_mask =
+		_mm_set_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED);
+	const __m128i mbuf_init =
+		_mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer);
+	__m128i rearm0, rearm1, rearm2, rearm3;
+
+	/* Extract pkt_info field. */
+	pinfo0 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+	pinfo = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	/* Extract hdr_type_etc field. */
+	pinfo0 = _mm_unpackhi_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpackhi_epi32(cqes[2], cqes[3]);
+	ptype = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	if (rxq->mark) {
+		const __m128i pinfo_ft_mask =
+			_mm_set_epi32(0xffffff00, 0xffffff00,
+				      0xffffff00, 0xffffff00);
+		const __m128i fdir_flags = _mm_set1_epi32(PKT_RX_FDIR);
+		const __m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID);
+		__m128i flow_tag, invalid_mask;
+
+		flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask);
+		/* Check if flow tag is non-zero then set PKT_RX_FDIR. */
+		invalid_mask = _mm_cmpeq_epi32(flow_tag, zero);
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(invalid_mask,
+							 fdir_flags));
+		/* Mask out invalid entries. */
+		flow_tag = _mm_andnot_si128(invalid_mask, flow_tag);
+		/* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(
+						_mm_cmpeq_epi32(flow_tag,
+								pinfo_ft_mask),
+						fdir_id_flags));
+	}
+	/*
+	 * Merge the two fields to generate the following:
+	 * bit[1]  = l3_ok,    bit[2]     = l4_ok
+	 * bit[8]  = cv,       bit[11:10] = l3_hdr_type
+	 * bit[12] = tunneled, bit[13]    = outer_l3_type
+	 */
+	ptype = _mm_and_si128(ptype, ptype_mask);
+	pinfo = _mm_and_si128(pinfo, pinfo_mask);
+	pinfo = _mm_slli_epi32(pinfo, 12);
+	ptype = _mm_or_si128(ptype, pinfo);
+	ptype = _mm_srli_epi32(ptype, 10);
+	ptype = _mm_packs_epi32(ptype, zero);
+	/* Errored packets will have RTE_PTYPE_ALL_MASK. */
+	op_err = _mm_srli_epi16(op_err, 12);
+	ptype = _mm_or_si128(ptype, op_err);
+	pkts[0]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 0)];
+	pkts[1]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 2)];
+	pkts[2]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 4)];
+	pkts[3]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 6)];
+	/* Fill flags for checksum and VLAN. */
+	pinfo = _mm_and_si128(pinfo, ptype_ol_mask);
+	pinfo = _mm_shuffle_epi8(cv_flag_sel, pinfo);
+	/* Locate checksum flags at byte[2:1] and merge with VLAN flags. */
+	cv_flags = _mm_slli_epi32(pinfo, 9);
+	cv_flags = _mm_or_si128(pinfo, cv_flags);
+	/* Move back flags to start from byte[0]. */
+	cv_flags = _mm_srli_epi32(cv_flags, 8);
+	/* Mask out garbage bits. */
+	cv_flags = _mm_and_si128(cv_flags, cv_mask);
+	/* Merge to ol_flags. */
+	ol_flags = _mm_or_si128(ol_flags, cv_flags);
+	/* Merge mbuf_init and ol_flags. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+			 offsetof(struct rte_mbuf, rearm_data) + 8);
+	rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 8), 0x30);
+	rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 4), 0x30);
+	rearm2 = _mm_blend_epi16(mbuf_init, ol_flags, 0x30);
+	rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(ol_flags, 4), 0x30);
+	/* Write 8B rearm_data and 8B ol_flags. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+			 RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+	_mm_store_si128((__m128i *)&pkts[0]->rearm_data, rearm0);
+	_mm_store_si128((__m128i *)&pkts[1]->rearm_data, rearm1);
+	_mm_store_si128((__m128i *)&pkts[2]->rearm_data, rearm2);
+	_mm_store_si128((__m128i *)&pkts[3]->rearm_data, rearm3);
+}
+
+/**
+ * Skip error packets.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+static uint16_t
+rxq_handle_pending_error(struct rxq *rxq, struct rte_mbuf **pkts,
+			 uint16_t pkts_n)
+{
+	uint16_t n = 0;
+	unsigned int i;
+
+	for (i = 0; i < pkts_n; ++i) {
+		struct rte_mbuf *pkt = pkts[i];
+
+		if (pkt->packet_type == RTE_PTYPE_ALL_MASK)
+			rte_pktmbuf_free_seg(pkt);
+		else
+			pkts[n++] = pkt;
+	}
+	rxq->stats.idropped += (pkts_n - n);
+	rxq->pending_err = 0;
+	return n;
+}
+
+/**
+ * Receive burst of packets. An errored CQE also consumes a mbuf, but the
+ * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed
+ * before returning to application.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets received including errors (<= pkts_n).
+ */
+static inline uint16_t
+rxq_burst_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	const uint16_t q_n = 1 << rxq->cqe_n;
+	const uint16_t q_mask = q_n - 1;
+	volatile struct mlx5_cqe *cq;
+	struct rte_mbuf **elts;
+	unsigned int pos;
+	uint64_t n;
+	uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
+	uint16_t nocmp_n = 0;
+	uint16_t rcvd_pkt = 0;
+	unsigned int cq_idx = rxq->cq_ci & q_mask;
+	unsigned int elts_idx;
+	unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1));
+	const __m128i owner_check =
+		_mm_set_epi64x(0x0100000001000000LL, 0x0100000001000000LL);
+	const __m128i opcode_check =
+		_mm_set_epi64x(0xf0000000f0000000LL, 0xf0000000f0000000LL);
+	const __m128i format_check =
+		_mm_set_epi64x(0x0c0000000c000000LL, 0x0c0000000c000000LL);
+	const __m128i resp_err_check =
+		_mm_set_epi64x(0xe0000000e0000000LL, 0xe0000000e0000000LL);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     12, 13,  8,  9,
+			      4,  5,  0,  1);
+#endif
+	/* Mask to shuffle from extracted CQE to mbuf. */
+	const __m128i shuf_mask =
+		_mm_set_epi8(-1,  3,  2,  1, /* fdir.hi */
+			     12, 13, 14, 15, /* rss, bswap32 */
+			     10, 11,         /* vlan_tci, bswap16 */
+			      4,  5,         /* data_len, bswap16 */
+			     -1, -1,         /* zero out 2nd half of pkt_len */
+			      4,  5          /* pkt_len, bswap16 */);
+	/* Mask to blend from the last Qword to the first DQword. */
+	const __m128i blend_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			      0,  0,  0,  0,
+			      0,  0,  0, -1);
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0);
+
+	/* Compile time sanity check for this function. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, pkt_info) != 0);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rx_hash_res) !=
+			 offsetof(struct mlx5_cqe, pkt_info) + 12);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd1) +
+			  sizeof(((struct mlx5_cqe *)0)->rsvd1) !=
+			 offsetof(struct mlx5_cqe, hdr_type_etc));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, vlan_info) !=
+			 offsetof(struct mlx5_cqe, hdr_type_etc) + 2);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd2) +
+			  sizeof(((struct mlx5_cqe *)0)->rsvd2) !=
+			 offsetof(struct mlx5_cqe, byte_cnt));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, sop_drop_qpn) !=
+			 RTE_ALIGN(offsetof(struct mlx5_cqe, sop_drop_qpn), 8));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, op_own) !=
+			 offsetof(struct mlx5_cqe, sop_drop_qpn) + 7);
+	assert(rxq->sges_n == 0);
+	assert(rxq->cqe_n == rxq->elts_n);
+	cq = &(*rxq->cqes)[cq_idx];
+	rte_prefetch0(cq);
+	rte_prefetch0(cq + 1);
+	rte_prefetch0(cq + 2);
+	rte_prefetch0(cq + 3);
+	pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
+	/*
+	 * Order of indexes:
+	 *   rq_ci >= cq_ci >= rq_pi
+	 * Definition of indexes:
+	 *   rq_ci - cq_ci := # of buffers owned by HW (posted).
+	 *   cq_ci - rq_pi := # of buffers not returned to app (decompressed).
+	 *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
+	 */
+	if ((uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)) >=
+			MLX5_VPMD_RXQ_RPLNSH_THRESH)
+		rxq_replenish_bulk_mbuf(rxq);
+	/* See if there're unreturned mbufs from compressed CQE. */
+	rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
+	if (rcvd_pkt > 0) {
+		rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
+		rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt);
+		rxq->rq_pi += rcvd_pkt;
+		pkts += rcvd_pkt;
+	}
+	elts_idx = rxq->rq_pi & q_mask;
+	elts = &(*rxq->elts)[elts_idx];
+	/* Not to overflow pkts array. */
+	pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
+	if (!pkts_n)
+		return rcvd_pkt;
+	/* At this point, there shouldn't be any remained packets. */
+	assert(rxq->rq_pi == rxq->cq_ci);
+	/*
+	 * A. load first Qword (8bytes) in one loop.
+	 * B. copy 4 mbuf pointers from elts ring to returing pkts.
+	 * C. load remained CQE data and extract necessary fields.
+	 *    Final 16bytes cqes[] extracted from original 64bytes CQE has the
+	 *    following structure:
+	 *        struct {
+	 *          uint8_t  pkt_info;
+	 *          uint8_t  flow_tag[3];
+	 *          uint16_t byte_cnt;
+	 *          uint8_t  rsvd4;
+	 *          uint8_t  op_own;
+	 *          uint16_t hdr_type_etc;
+	 *          uint16_t vlan_info;
+	 *          uint32_t rx_has_res;
+	 *        } c;
+	 * D. fill in mbuf.
+	 * E. get valid CQEs.
+	 * F. find compressed CQE.
+	 */
+	for (pos = 0;
+	     pos < pkts_n;
+	     pos += MLX5_VPMD_DESCS_PER_LOOP) {
+		__m128i cqes[MLX5_VPMD_DESCS_PER_LOOP];
+		__m128i cqe_tmp1, cqe_tmp2;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
+		__m128i op_own, op_own_tmp1, op_own_tmp2;
+		__m128i opcode, owner_mask, invalid_mask;
+		__m128i comp_mask;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt;
+#endif
+		__m128i mbp1, mbp2;
+		__m128i p = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0);
+		unsigned int p1, p2, p3;
+
+		/* Prefetch next 4 CQEs. */
+		if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) {
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]);
+		}
+		/* A.0 Do not cross the end of CQ. */
+		p = _mm_andnot_si128(ones << ((pkts_n - pos) *
+					      sizeof(uint16_t) * 8), p);
+		/* A.1 load cqes. */
+		p3 = _mm_extract_epi16(p, 3);
+		cqes[3] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p3].sop_drop_qpn);
+		rte_compiler_barrier();
+		p2 = _mm_extract_epi16(p, 2);
+		cqes[2] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p2].sop_drop_qpn);
+		rte_compiler_barrier();
+		/* B.1 load mbuf pointers. */
+		mbp1 = _mm_loadu_si128((__m128i *)&elts[pos]);
+		mbp2 = _mm_loadu_si128((__m128i *)&elts[pos + 2]);
+		/* A.1 load a block having op_own. */
+		p1 = _mm_extract_epi16(p, 1);
+		cqes[1] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p1].sop_drop_qpn);
+		rte_compiler_barrier();
+		cqes[0] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos].sop_drop_qpn);
+		/* B.2 copy mbuf pointers. */
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp1);
+		_mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2);
+		rte_compiler_barrier();
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]);
+		cqes[3] = _mm_blendv_epi8(cqes[3], cqe_tmp2, blend_mask);
+		cqes[2] = _mm_blendv_epi8(cqes[2], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p3].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos + p2].rsvd1[3]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x30);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p3].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos + p2].rsvd2[10]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x04);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb3 = _mm_shuffle_epi8(cqes[3], shuf_mask);
+		pkt_mb2 = _mm_shuffle_epi8(cqes[2], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb3 = _mm_sub_epi16(pkt_mb3, crc_adj);
+		pkt_mb2 = _mm_sub_epi16(pkt_mb2, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb3 = _mm_add_epi32(pkt_mb3, flow_mark_adj);
+		pkt_mb2 = _mm_add_epi32(pkt_mb2, flow_mark_adj);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 3]->pkt_len, pkt_mb3);
+		_mm_storeu_si128((void *)&pkts[pos + 2]->pkt_len, pkt_mb2);
+		/* E.1 extract op_own field. */
+		op_own_tmp2 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p1]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos]);
+		cqes[1] = _mm_blendv_epi8(cqes[1], cqe_tmp2, blend_mask);
+		cqes[0] = _mm_blendv_epi8(cqes[0], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p1].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos].rsvd1[3]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x30);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p1].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos].rsvd2[10]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x04);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb1 = _mm_shuffle_epi8(cqes[1], shuf_mask);
+		pkt_mb0 = _mm_shuffle_epi8(cqes[0], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb1 = _mm_sub_epi16(pkt_mb1, crc_adj);
+		pkt_mb0 = _mm_sub_epi16(pkt_mb0, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb1 = _mm_add_epi32(pkt_mb1, flow_mark_adj);
+		pkt_mb0 = _mm_add_epi32(pkt_mb0, flow_mark_adj);
+		/* E.1 extract op_own byte. */
+		op_own_tmp1 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+		op_own = _mm_unpackhi_epi64(op_own_tmp1, op_own_tmp2);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 1]->pkt_len, pkt_mb1);
+		_mm_storeu_si128((void *)&pkts[pos]->pkt_len, pkt_mb0);
+		/* E.2 flip owner bit to mark CQEs from last round. */
+		owner_mask = _mm_and_si128(op_own, owner_check);
+		if (ownership)
+			owner_mask = _mm_xor_si128(owner_mask, owner_check);
+		owner_mask = _mm_cmpeq_epi32(owner_mask, owner_check);
+		owner_mask = _mm_packs_epi32(owner_mask, zero);
+		/* E.3 get mask for invalidated CQEs. */
+		opcode = _mm_and_si128(op_own, opcode_check);
+		invalid_mask = _mm_cmpeq_epi32(opcode_check, opcode);
+		invalid_mask = _mm_packs_epi32(invalid_mask, zero);
+		/* E.4 mask out beyond boundary. */
+		invalid_mask =
+			_mm_or_si128(invalid_mask,
+				     ones << ((pkts_n - pos) *
+					      sizeof(uint16_t) * 8));
+		/* E.5 merge invalid_mask with invalid owner. */
+		invalid_mask = _mm_or_si128(invalid_mask, owner_mask);
+		/* F.1 find compressed CQE format. */
+		comp_mask = _mm_and_si128(op_own, format_check);
+		comp_mask = _mm_cmpeq_epi32(comp_mask, format_check);
+		comp_mask = _mm_packs_epi32(comp_mask, zero);
+		/* F.2 mask out invalid entries. */
+		comp_mask = _mm_andnot_si128(invalid_mask, comp_mask);
+		comp_idx = _mm_cvtsi128_si64(comp_mask);
+		/* F.3 get the first compressed CQE. */
+		comp_idx = comp_idx ?
+				__builtin_ctzll(comp_idx) /
+					(sizeof(uint16_t) * 8) :
+				MLX5_VPMD_DESCS_PER_LOOP;
+		/* E.6 mask out entries after the compressed CQE. */
+		invalid_mask =
+			_mm_or_si128(invalid_mask,
+				     ones << (comp_idx *
+					      sizeof(uint16_t) * 8));
+		/* E.7 count non-compressed valid CQEs. */
+		n = _mm_cvtsi128_si64(invalid_mask);
+		n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) :
+			MLX5_VPMD_DESCS_PER_LOOP;
+		nocmp_n += n;
+		/* D.2 get the final invalid mask. */
+		invalid_mask =
+			_mm_or_si128(invalid_mask,
+				     ones << (n * sizeof(uint16_t) * 8));
+		/* D.3 check error in opcode. */
+		opcode = _mm_cmpeq_epi32(resp_err_check, opcode);
+		opcode = _mm_packs_epi32(opcode, zero);
+		opcode = _mm_andnot_si128(invalid_mask, opcode);
+		/* D.4 mark if any error is set */
+		rxq->pending_err |= !!_mm_cvtsi128_si64(opcode);
+		/* D.5 fill in mbuf - rearm_data and packet_type. */
+		rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Add up received bytes count. */
+		byte_cnt = _mm_shuffle_epi8(op_own, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		/*
+		 * Break the loop unless more valid CQE is expected, or if
+		 * there's a compressed CQE.
+		 */
+		if (n != MLX5_VPMD_DESCS_PER_LOOP)
+			break;
+	}
+	/* If no new CQE seen, return without updating cq_db. */
+	if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP))
+		return rcvd_pkt;
+	/* Update the consumer indexes for non-compressed CQEs. */
+	assert(nocmp_n <= pkts_n);
+	rxq->cq_ci += nocmp_n;
+	rxq->rq_pi += nocmp_n;
+	rcvd_pkt += nocmp_n;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += nocmp_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	/* Decompress the last CQE if compressed. */
+	if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) {
+		assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
+		rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]);
+		/* Return more packets if needed. */
+		if (nocmp_n < pkts_n) {
+			uint16_t n = rxq->cq_ci - rxq->rq_pi;
+
+			n = RTE_MIN(n, pkts_n - nocmp_n);
+			rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n);
+			rxq->rq_pi += n;
+			rcvd_pkt += n;
+		}
+	}
+	rte_wmb();
+	*rxq->cq_db = htonl(rxq->cq_ci);
+	return rcvd_pkt;
+}
+
+/**
+ * DPDK callback for vectorized RX.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct rxq *rxq = dpdk_rxq;
+	uint16_t nb_rx;
+
+	nb_rx = rxq_burst_v(rxq, pkts, pkts_n);
+	if (unlikely(rxq->pending_err))
+		nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
+	return nb_rx;
+}
+
+/**
+ * Check a TX queue can support vectorized TX.
+ *
+ * @param txq
+ *   Pointer to TX queue.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+txq_check_vec_tx_support(struct txq *txq)
+{
+	/* Currently unused, but for possible future use. */
+	(void)txq;
+	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check Tx queue flags are set for raw vectorized Tx.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_raw_vec_tx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->txqs_n; ++i) {
+		struct txq *txq = (*priv->txqs)[i];
+
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) ||
+		    !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
+			break;
+	}
+	if (i != priv->txqs_n)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized TX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_vec_tx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->txqs_n; ++i) {
+		struct txq *txq = (*priv->txqs)[i];
+
+		if (txq_check_vec_tx_support(txq) < 0)
+			break;
+	}
+	if (i != priv->txqs_n)
+		return -ENOTSUP;
+	if (priv->txqs_n > MLX5_VPMD_MIN_TXQS ||
+	    priv->mps != MLX5_MPW_ENHANCED ||
+	    priv->tso)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a RX queue can support vectorized RX.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+rxq_check_vec_rx_support(struct rxq *rxq)
+{
+	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1) ||
+	    rxq->sges_n != 0)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized RX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_vec_rx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+
+		if (rxq_check_vec_rx_support(rxq) < 0)
+			break;
+	}
+	if (i != priv->rxqs_n)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Prepare for vectorized RX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+void
+priv_prep_vec_rx_function(struct priv *priv)
+{
+	uint16_t i;
+
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+		struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
+		const uint16_t desc = 1 << rxq->elts_n;
+		int j;
+
+		assert(rxq->elts_n == rxq->cqe_n);
+		/* Initialize default rearm_data for vPMD. */
+		mbuf_init->data_off = RTE_PKTMBUF_HEADROOM;
+		rte_mbuf_refcnt_set(mbuf_init, 1);
+		mbuf_init->nb_segs = 1;
+		mbuf_init->port = rxq->port_id;
+		/*
+		 * prevent compiler reordering:
+		 * rearm_data covers previous fields.
+		 */
+		rte_compiler_barrier();
+		rxq->mbuf_initializer =
+			*(uint64_t *)&mbuf_init->rearm_data;
+		/* Padding with a fake mbuf for vectorized Rx. */
+		for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j)
+			(*rxq->elts)[desc + j] = &rxq->fake_mbuf;
+		/* Mark that it need to be cleaned up for rxq_alloc_elts(). */
+		rxq->trim_elts = 1;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index c2d6c303d..682054108 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -244,7 +244,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
 		ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set");
 		goto error;
 	}
-	(void)conf; /* Thresholds configuration (ignored). */
+	tmpl.txq.flags = conf->txq_flags;
 	assert(desc > MLX5_TX_COMP_THRESH);
 	tmpl.txq.elts_n = log2above(desc);
 	if (priv->mps == MLX5_MPW_ENHANCED)
-- 
2.11.0

[dpdk-dev] [PATCH v2 0/5] net/mlx5: add vectorized Rx/Tx burst for x86

[Yongseok Koh]

This is to introduce more efficient Rx/Tx burst functions using SIMD
instructions. Currently it is only supported by 64bit x86 having SSE4.1.

>From functional perspective, Rx burst function is equivalent to the
existing mlx5_rx_burst() except for scatter support, which will be added
soon. Tx burst function supports multi-segment packets and offload flags
unless it is disabled by txq_flags. However, disabling those features will
bring a little higher performance.

v2:
* Streamline redundant conditional clauses in txq_complete().
* Remove the mempool pointer in txq->mp2mr structure.
* Fix indentation and spacing.

Yongseok Koh (5):
  net/mlx5: change indexing for Tx SW ring
  net/mlx5: free buffers in bulk on Tx completion
  net/mlx5: use buffer address for LKEY search
  net/mlx5: select Rx/Tx callbacks when starting device
  net/mlx5: add vectorized Rx/Tx burst for SSE4.1

 drivers/net/mlx5/Makefile            |   10 +
 drivers/net/mlx5/mlx5_defs.h         |   18 +
 drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
 drivers/net/mlx5/mlx5_mr.c           |   17 +-
 drivers/net/mlx5/mlx5_rxq.c          |   57 +-
 drivers/net/mlx5/mlx5_rxtx.c         |  425 +++-------
 drivers/net/mlx5/mlx5_rxtx.h         |  291 ++++++-
 drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_trigger.c      |    3 +
 drivers/net/mlx5/mlx5_txq.c          |   23 +-
 10 files changed, 1980 insertions(+), 343 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c

-- 
2.11.0

[dpdk-dev] [PATCH v2 1/5] net/mlx5: change indexing for Tx SW ring

[Yongseok Koh]

For Tx SW ring (txq->elts[]), indexes are kept and used in
txq->elts_head/tail. Because of this, one entry must always be left unused
and it also makes code complex. Changed to store counters instead of
indexes in order to make the code simpler and to reduce a few calculations.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 108 +++++++++++++++++--------------------------
 drivers/net/mlx5/mlx5_rxtx.h |   4 +-
 drivers/net/mlx5/mlx5_txq.c  |  16 +++----
 3 files changed, 52 insertions(+), 76 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 70314b393..43db06ad8 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -255,7 +255,8 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
 static inline void
 txq_complete(struct txq *txq)
 {
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	const unsigned int cqe_n = 1 << txq->cqe_n;
 	const unsigned int cqe_cnt = cqe_n - 1;
 	uint16_t elts_free = txq->elts_tail;
@@ -292,24 +293,23 @@ txq_complete(struct txq *txq)
 	ctrl = (volatile struct mlx5_wqe_ctrl *)
 		tx_mlx5_wqe(txq, txq->wqe_pi);
 	elts_tail = ctrl->ctrl3;
-	assert(elts_tail < (1 << txq->wqe_n));
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
 	/* Free buffers. */
 	while (elts_free != elts_tail) {
-		struct rte_mbuf *elt = (*txq->elts)[elts_free];
-		unsigned int elts_free_next =
-			(elts_free + 1) & (elts_n - 1);
-		struct rte_mbuf *elt_next = (*txq->elts)[elts_free_next];
+		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
+		struct rte_mbuf *elt_next =
+			(*txq->elts)[(elts_free + 1) & elts_m];
 
 #ifndef NDEBUG
 		/* Poisoning. */
-		memset(&(*txq->elts)[elts_free],
+		memset(&(*txq->elts)[elts_free & elts_m],
 		       0x66,
-		       sizeof((*txq->elts)[elts_free]));
+		       sizeof((*txq->elts)[elts_free & elts_m]));
 #endif
 		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
 		/* Only one segment needs to be freed. */
 		rte_pktmbuf_free_seg(elt);
-		elts_free = elts_free_next;
+		++elts_free;
 	}
 	txq->cq_ci = cq_ci;
 	txq->elts_tail = elts_tail;
@@ -409,12 +409,10 @@ int
 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
 {
 	struct txq *txq = tx_queue;
-	const unsigned int elts_n = 1 << txq->elts_n;
-	const unsigned int elts_cnt = elts_n - 1;
-	unsigned int used;
+	uint16_t used;
 
 	txq_complete(txq);
-	used = (txq->elts_head - txq->elts_tail) & elts_cnt;
+	used = txq->elts_head - txq->elts_tail;
 	if (offset < used)
 		return RTE_ETH_TX_DESC_FULL;
 	return RTE_ETH_TX_DESC_DONE;
@@ -488,11 +486,12 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
 	unsigned int k = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	unsigned int max_inline = txq->max_inline;
 	const unsigned int inline_en = !!max_inline && txq->inline_en;
 	uint16_t max_wqe;
@@ -509,9 +508,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(*pkts);
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
@@ -540,9 +537,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--segs_n;
 		if (unlikely(--max_wqe == 0))
 			break;
@@ -561,7 +558,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		if (length < (MLX5_WQE_DWORD_SIZE + 2))
 			break;
 		/* Update element. */
-		(*txq->elts)[elts_head] = buf;
+		(*txq->elts)[elts_head & elts_m] = buf;
 		/* Prefetch next buffer data. */
 		if (pkts_n - i > 1)
 			rte_prefetch0(
@@ -801,8 +798,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr,
 			naddr >> 32,
 		};
-		elts_head = (elts_head + 1) & (elts_n - 1);
-		(*txq->elts)[elts_head] = buf;
+		(*txq->elts)[++elts_head & elts_m] = buf;
 		++sg;
 		/* Advance counter only if all segs are successfully posted. */
 		if (sg < segs_n)
@@ -810,7 +806,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		else
 			j += sg;
 next_pkt:
-		elts_head = (elts_head + 1) & (elts_n - 1);
+		++elts_head;
 		++pkts;
 		++i;
 		/* Initialize known and common part of the WQE structure. */
@@ -853,7 +849,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Take a shortcut if nothing must be sent. */
 	if (unlikely((i + k) == 0))
 		return 0;
-	txq->elts_head = (txq->elts_head + i + j) & (elts_n - 1);
+	txq->elts_head += (i + j);
 	/* Check whether completion threshold has been reached. */
 	comp = txq->elts_comp + i + j + k;
 	if (comp >= MLX5_TX_COMP_THRESH) {
@@ -960,10 +956,11 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int comp;
 	struct mlx5_mpw mpw = {
@@ -977,15 +974,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
 		uint32_t cs_flags = 0;
@@ -995,12 +989,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--pkts_n;
 		/* Should we enable HW CKSUM offload */
 		if (buf->ol_flags &
@@ -1036,9 +1030,8 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			volatile struct mlx5_wqe_data_seg *dseg;
 			uintptr_t addr;
 
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
 			assert(buf);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			dseg = mpw.data.dseg[mpw.pkts_n];
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
@@ -1046,7 +1039,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 				.addr = htonll(addr),
 			};
-			elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 			length += DATA_LEN(buf);
 #endif
@@ -1057,7 +1049,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		assert(length == mpw.len);
 		if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
 			mlx5_mpw_close(txq, &mpw);
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
@@ -1175,10 +1166,11 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int comp;
 	unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE;
@@ -1205,12 +1197,9 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uintptr_t addr;
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
@@ -1221,12 +1210,12 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--pkts_n;
 		/*
 		 * Compute max_wqe in case less WQE were consumed in previous
@@ -1287,10 +1276,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 			do {
 				volatile struct mlx5_wqe_data_seg *dseg;
 
-				elts_head_next =
-					(elts_head + 1) & (elts_n - 1);
 				assert(buf);
-				(*txq->elts)[elts_head] = buf;
+				(*txq->elts)[elts_head++ & elts_m] = buf;
 				dseg = mpw.data.dseg[mpw.pkts_n];
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
@@ -1298,7 +1285,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 					.addr = htonll(addr),
 				};
-				elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 				length += DATA_LEN(buf);
 #endif
@@ -1315,9 +1301,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 			assert(mpw.state == MLX5_MPW_INL_STATE_OPENED);
 			assert(length <= inline_room);
 			assert(length == DATA_LEN(buf));
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			/* Maximum number of bytes before wrapping. */
 			max = ((((uintptr_t)(txq->wqes)) +
 				(1 << txq->wqe_n) *
@@ -1354,7 +1339,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				inline_room -= length;
 			}
 		}
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
@@ -1476,10 +1460,11 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max_elts;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE;
 	unsigned int mpw_room = 0;
@@ -1494,8 +1479,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Start processing. */
 	txq_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	if (max_elts > elts_n)
-		max_elts -= elts_n;
 	/* A CQE slot must always be available. */
 	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
@@ -1503,7 +1486,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		return 0;
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uintptr_t addr;
 		uint64_t naddr;
 		unsigned int n;
@@ -1517,7 +1499,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max_elts - j < segs_n + 1)
+		if (max_elts - j < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
@@ -1601,10 +1583,8 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			do {
 				volatile struct mlx5_wqe_data_seg *dseg;
 
-				elts_head_next =
-					(elts_head + 1) & (elts_n - 1);
 				assert(buf);
-				(*txq->elts)[elts_head] = buf;
+				(*txq->elts)[elts_head++ & elts_m] = buf;
 				dseg = mpw.data.dseg[mpw.pkts_n];
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
@@ -1612,7 +1592,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 					.addr = htonll(addr),
 				};
-				elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 				length += DATA_LEN(buf);
 #endif
@@ -1663,7 +1642,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			/* No need to get completion as the entire packet is
 			 * copied to WQ. Free the buf right away.
 			 */
-			elts_head_next = elts_head;
 			rte_pktmbuf_free_seg(buf);
 			mpw_room -= (inl_pad + sizeof(inl_hdr) + length);
 			/* Add pad in the next packet if any. */
@@ -1686,8 +1664,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				dseg = (volatile void *)
 					((uintptr_t)mpw.data.raw +
 					 inl_pad);
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			for (n = 0; n * RTE_CACHE_LINE_SIZE < length; n++)
 				rte_prefetch2((void *)(addr +
@@ -1706,7 +1683,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			mpw_room -= (inl_pad + sizeof(*dseg));
 			inl_pad = 0;
 		}
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 450a56966..f9b738b4e 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -242,8 +242,8 @@ struct hash_rxq {
 /* TX queue descriptor. */
 RTE_STD_C11
 struct txq {
-	uint16_t elts_head; /* Current index in (*elts)[]. */
-	uint16_t elts_tail; /* First element awaiting completion. */
+	uint16_t elts_head; /* Current counter in (*elts)[]. */
+	uint16_t elts_tail; /* Counter of first element awaiting completion. */
 	uint16_t elts_comp; /* Counter since last completion request. */
 	uint16_t mpw_comp; /* WQ index since last completion request. */
 	uint16_t cq_ci; /* Consumer index for completion queue. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index bf72468d5..f0729a2a8 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -103,9 +103,10 @@ txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n)
 static void
 txq_free_elts(struct txq_ctrl *txq_ctrl)
 {
-	unsigned int elts_n = 1 << txq_ctrl->txq.elts_n;
-	unsigned int elts_head = txq_ctrl->txq.elts_head;
-	unsigned int elts_tail = txq_ctrl->txq.elts_tail;
+	const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	uint16_t elts_head = txq_ctrl->txq.elts_head;
+	uint16_t elts_tail = txq_ctrl->txq.elts_tail;
 	struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts;
 
 	DEBUG("%p: freeing WRs", (void *)txq_ctrl);
@@ -114,18 +115,17 @@ txq_free_elts(struct txq_ctrl *txq_ctrl)
 	txq_ctrl->txq.elts_comp = 0;
 
 	while (elts_tail != elts_head) {
-		struct rte_mbuf *elt = (*elts)[elts_tail];
+		struct rte_mbuf *elt = (*elts)[elts_tail & elts_m];
 
 		assert(elt != NULL);
 		rte_pktmbuf_free_seg(elt);
 #ifndef NDEBUG
 		/* Poisoning. */
-		memset(&(*elts)[elts_tail],
+		memset(&(*elts)[elts_tail & elts_m],
 		       0x77,
-		       sizeof((*elts)[elts_tail]));
+		       sizeof((*elts)[elts_tail & elts_m]));
 #endif
-		if (++elts_tail == elts_n)
-			elts_tail = 0;
+		++elts_tail;
 	}
 }
 
-- 
2.11.0

Re: [dpdk-dev] [PATCH v2 1/5] net/mlx5: change indexing for Tx SW ring

[Nélio Laranjeiro]

On Fri, Jun 30, 2017 at 12:23:29PM -0700, Yongseok Koh wrote:
> For Tx SW ring (txq->elts[]), indexes are kept and used in
> txq->elts_head/tail. Because of this, one entry must always be left unused
> and it also makes code complex. Changed to store counters instead of
> indexes in order to make the code simpler and to reduce a few calculations.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---

Already acked in V1, please keep the ack in the unchanged commits.

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 2/5] net/mlx5: free buffers in bulk on Tx completion

[Yongseok Koh]

When processing Tx completion, it is more efficient to free buffers in bulk
using rte_mempool_put_bulk() if buffers are from a same mempool.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 43db06ad8..66593679f 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -264,6 +264,9 @@ txq_complete(struct txq *txq)
 	uint16_t cq_ci = txq->cq_ci;
 	volatile struct mlx5_cqe *cqe = NULL;
 	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	struct rte_mempool *pool = NULL;
+	unsigned int blk_n = 0;
 
 	do {
 		volatile struct mlx5_cqe *tmp;
@@ -296,21 +299,33 @@ txq_complete(struct txq *txq)
 	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
 	/* Free buffers. */
 	while (elts_free != elts_tail) {
-		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
-		struct rte_mbuf *elt_next =
-			(*txq->elts)[(elts_free + 1) & elts_m];
-
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (likely(m->pool == pool)) {
+				free[blk_n++] = m;
+			} else {
+				if (likely(pool != NULL))
+					rte_mempool_put_bulk(pool,
+							     (void *)free,
+							     blk_n);
+				free[0] = m;
+				pool = m->pool;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(pool, (void *)free, blk_n);
 #ifndef NDEBUG
-		/* Poisoning. */
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
 		memset(&(*txq->elts)[elts_free & elts_m],
 		       0x66,
 		       sizeof((*txq->elts)[elts_free & elts_m]));
-#endif
-		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
-		/* Only one segment needs to be freed. */
-		rte_pktmbuf_free_seg(elt);
 		++elts_free;
 	}
+#endif
 	txq->cq_ci = cq_ci;
 	txq->elts_tail = elts_tail;
 	/* Update the consumer index. */
-- 
2.11.0

Re: [dpdk-dev] [PATCH v2 2/5] net/mlx5: free buffers in bulk on Tx completion

[Nélio Laranjeiro]

On Fri, Jun 30, 2017 at 12:23:30PM -0700, Yongseok Koh wrote:
> When processing Tx completion, it is more efficient to free buffers in bulk
> using rte_mempool_put_bulk() if buffers are from a same mempool.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
>[...]

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 3/5] net/mlx5: use buffer address for LKEY search

[Yongseok Koh]

When searching LKEY, if search key is mempool pointer, the 2nd cacheline
has to be accessed and it even requires to check whether a buffer is
indirect per every search. Instead, using address for search key can reduce
cycles taken. And caching the last hit entry is beneficial as well.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_mr.c   | 17 ++++++++++++++---
 drivers/net/mlx5/mlx5_rxtx.c | 39 +++++++++++++++++++++------------------
 drivers/net/mlx5/mlx5_rxtx.h |  4 +++-
 drivers/net/mlx5/mlx5_txq.c  |  3 +--
 4 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 0a3638460..287335179 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -207,7 +207,8 @@ txq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx)
 			 sizeof(txq_ctrl->txq.mp2mr[0])));
 	}
 	/* Store the new entry. */
-	txq_ctrl->txq.mp2mr[idx].mp = mp;
+	txq_ctrl->txq.mp2mr[idx].start = (uintptr_t)mr->addr;
+	txq_ctrl->txq.mp2mr[idx].end = (uintptr_t)mr->addr + mr->length;
 	txq_ctrl->txq.mp2mr[idx].mr = mr;
 	txq_ctrl->txq.mp2mr[idx].lkey = htonl(mr->lkey);
 	DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
@@ -265,18 +266,28 @@ txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
 	struct txq_mp2mr_mbuf_check_data data = {
 		.ret = 0,
 	};
+	uintptr_t start;
+	uintptr_t end;
 	unsigned int i;
 
 	/* Register mempool only if the first element looks like a mbuf. */
 	if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
 			data.ret == -1)
 		return;
+	if (mlx5_check_mempool(mp, &start, &end) != 0) {
+		ERROR("mempool %p: not virtually contiguous",
+		      (void *)mp);
+		return;
+	}
 	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
-		if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) {
+		struct ibv_mr *mr = txq_ctrl->txq.mp2mr[i].mr;
+
+		if (unlikely(mr == NULL)) {
 			/* Unknown MP, add a new MR for it. */
 			break;
 		}
-		if (txq_ctrl->txq.mp2mr[i].mp == mp)
+		if (start >= (uintptr_t)mr->addr &&
+		    end <= (uintptr_t)mr->addr + mr->length)
 			return;
 	}
 	txq_mp2mr_reg(&txq_ctrl->txq, mp, i);
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 66593679f..688ee9028 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -77,7 +77,7 @@ static __rte_always_inline void
 txq_complete(struct txq *txq);
 
 static __rte_always_inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp);
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb);
 
 static __rte_always_inline void
 mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe);
@@ -352,7 +352,7 @@ txq_mb2mp(struct rte_mbuf *buf)
 }
 
 /**
- * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[].
+ * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
  * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
  * remove an entry first.
  *
@@ -365,27 +365,30 @@ txq_mb2mp(struct rte_mbuf *buf)
  *   mr->lkey on success, (uint32_t)-1 on failure.
  */
 static inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
 {
-	unsigned int i;
-	uint32_t lkey = (uint32_t)-1;
+	uint16_t i = txq->mr_cache_idx;
+	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
 
+	assert(i < RTE_DIM(txq->mp2mr));
+	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
+		return txq->mp2mr[i].lkey;
 	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
-		if (unlikely(txq->mp2mr[i].mp == NULL)) {
+		if (unlikely(txq->mp2mr[i].mr == NULL)) {
 			/* Unknown MP, add a new MR for it. */
 			break;
 		}
-		if (txq->mp2mr[i].mp == mp) {
+		if (txq->mp2mr[i].start <= addr &&
+		    txq->mp2mr[i].end >= addr) {
 			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
 			assert(htonl(txq->mp2mr[i].mr->lkey) ==
 			       txq->mp2mr[i].lkey);
-			lkey = txq->mp2mr[i].lkey;
-			break;
+			txq->mr_cache_idx = i;
+			return txq->mp2mr[i].lkey;
 		}
 	}
-	if (unlikely(lkey == (uint32_t)-1))
-		lkey = txq_mp2mr_reg(txq, mp, i);
-	return lkey;
+	txq->mr_cache_idx = 0;
+	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
 }
 
 /**
@@ -770,7 +773,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t){
 				htonl(length),
-				txq_mp2mr(txq, txq_mb2mp(buf)),
+				txq_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
@@ -809,7 +812,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		naddr = htonll(rte_pktmbuf_mtod(buf, uintptr_t));
 		*dseg = (rte_v128u32_t){
 			htonl(length),
-			txq_mp2mr(txq, txq_mb2mp(buf)),
+			txq_mb2mr(txq, buf),
 			naddr,
 			naddr >> 32,
 		};
@@ -1051,7 +1054,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
 				.byte_count = htonl(DATA_LEN(buf)),
-				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+				.lkey = txq_mb2mr(txq, buf),
 				.addr = htonll(addr),
 			};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1297,7 +1300,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+					.lkey = txq_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1604,7 +1607,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+					.lkey = txq_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1687,7 +1690,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t) {
 				htonl(length),
-				txq_mp2mr(txq, txq_mb2mp(buf)),
+				txq_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index f9b738b4e..51e258a15 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -267,10 +267,12 @@ struct txq {
 	volatile uint32_t *cq_db; /* Completion queue doorbell. */
 	volatile void *bf_reg; /* Blueflame register. */
 	struct {
-		const struct rte_mempool *mp; /* Cached Memory Pool. */
+		uintptr_t start; /* Start address of MR */
+		uintptr_t end; /* End address of MR */
 		struct ibv_mr *mr; /* Memory Region (for mp). */
 		uint32_t lkey; /* htonl(mr->lkey) */
 	} mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MP to MR translation table. */
+	uint16_t mr_cache_idx; /* Index of last hit entry. */
 	struct rte_mbuf *(*elts)[]; /* TX elements. */
 	struct mlx5_txq_stats stats; /* TX queue counters. */
 } __rte_cache_aligned;
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index f0729a2a8..ac9dfc5f0 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -149,9 +149,8 @@ txq_cleanup(struct txq_ctrl *txq_ctrl)
 	if (txq_ctrl->cq != NULL)
 		claim_zero(ibv_destroy_cq(txq_ctrl->cq));
 	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
-		if (txq_ctrl->txq.mp2mr[i].mp == NULL)
+		if (txq_ctrl->txq.mp2mr[i].mr == NULL)
 			break;
-		assert(txq_ctrl->txq.mp2mr[i].mr != NULL);
 		claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr));
 	}
 	memset(txq_ctrl, 0, sizeof(*txq_ctrl));
-- 
2.11.0

Re: [dpdk-dev] [PATCH v2 3/5] net/mlx5: use buffer address for LKEY search

[Nélio Laranjeiro]

On Fri, Jun 30, 2017 at 12:23:31PM -0700, Yongseok Koh wrote:
> When searching LKEY, if search key is mempool pointer, the 2nd cacheline
> has to be accessed and it even requires to check whether a buffer is
> indirect per every search. Instead, using address for search key can reduce
> cycles taken. And caching the last hit entry is beneficial as well.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5_mr.c   | 17 ++++++++++++++---
>  drivers/net/mlx5/mlx5_rxtx.c | 39 +++++++++++++++++++++------------------
>  drivers/net/mlx5/mlx5_rxtx.h |  4 +++-
>  drivers/net/mlx5/mlx5_txq.c  |  3 +--
>  4 files changed, 39 insertions(+), 24 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> index 0a3638460..287335179 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -265,18 +266,28 @@ txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
>  	struct txq_mp2mr_mbuf_check_data data = {
>  		.ret = 0,
>  	};
> +	uintptr_t start;
> +	uintptr_t end;
>  	unsigned int i;
>  
>  	/* Register mempool only if the first element looks like a mbuf. */
>  	if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
>  			data.ret == -1)
>  		return;
> +	if (mlx5_check_mempool(mp, &start, &end) != 0) {
> +		ERROR("mempool %p: not virtually contiguous",
> +		      (void *)mp);
> +		return;
> +	}
>  	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
> -		if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) {
> +		struct ibv_mr *mr = txq_ctrl->txq.mp2mr[i].mr;
> +
> +		if (unlikely(mr == NULL)) {
>  			/* Unknown MP, add a new MR for it. */
>  			break;
>  		}
> -		if (txq_ctrl->txq.mp2mr[i].mp == mp)
> +		if (start >= (uintptr_t)mr->addr &&
> +		    end <= (uintptr_t)mr->addr + mr->length)
>  			return;
>  	}
>  	txq_mp2mr_reg(&txq_ctrl->txq, mp, i);

 if (start >= (uintptr_t)mr->addr &&
     end <= (uintptr_t)mr->addr + mr->length)

Is this expected to have a memory region bigger than the memory pool
space?  I mean I was expecting to see strict equality in the addresses.

Regards,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v2 3/5] net/mlx5: use buffer address for LKEY search

[Yongseok Koh]

> On Jul 3, 2017, at 7:06 AM, Nélio Laranjeiro <nelio.laranjeiro@6wind.com> wrote:
> 
> On Fri, Jun 30, 2017 at 12:23:31PM -0700, Yongseok Koh wrote:
>> When searching LKEY, if search key is mempool pointer, the 2nd cacheline
>> has to be accessed and it even requires to check whether a buffer is
>> indirect per every search. Instead, using address for search key can reduce
>> cycles taken. And caching the last hit entry is beneficial as well.
>> 
>> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
>> ---
>> drivers/net/mlx5/mlx5_mr.c   | 17 ++++++++++++++---
>> drivers/net/mlx5/mlx5_rxtx.c | 39 +++++++++++++++++++++------------------
>> drivers/net/mlx5/mlx5_rxtx.h |  4 +++-
>> drivers/net/mlx5/mlx5_txq.c  |  3 +--
>> 4 files changed, 39 insertions(+), 24 deletions(-)
>> 
>> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
>> index 0a3638460..287335179 100644
>> --- a/drivers/net/mlx5/mlx5_mr.c
>> +++ b/drivers/net/mlx5/mlx5_mr.c
>> @@ -265,18 +266,28 @@ txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
>> 	struct txq_mp2mr_mbuf_check_data data = {
>> 		.ret = 0,
>> 	};
>> +	uintptr_t start;
>> +	uintptr_t end;
>> 	unsigned int i;
>> 
>> 	/* Register mempool only if the first element looks like a mbuf. */
>> 	if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
>> 			data.ret == -1)
>> 		return;
>> +	if (mlx5_check_mempool(mp, &start, &end) != 0) {
>> +		ERROR("mempool %p: not virtually contiguous",
>> +		      (void *)mp);
>> +		return;
>> +	}
>> 	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
>> -		if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) {
>> +		struct ibv_mr *mr = txq_ctrl->txq.mp2mr[i].mr;
>> +
>> +		if (unlikely(mr == NULL)) {
>> 			/* Unknown MP, add a new MR for it. */
>> 			break;
>> 		}
>> -		if (txq_ctrl->txq.mp2mr[i].mp == mp)
>> +		if (start >= (uintptr_t)mr->addr &&
>> +		    end <= (uintptr_t)mr->addr + mr->length)
>> 			return;
>> 	}
>> 	txq_mp2mr_reg(&txq_ctrl->txq, mp, i);
> 
> if (start >= (uintptr_t)mr->addr &&
>     end <= (uintptr_t)mr->addr + mr->length)
> 
> Is this expected to have a memory region bigger than the memory pool
> space?  I mean I was expecting to see strict equality in the addresses.
In mlx5_mp2mr(), start/end of a memory region are rounded up to make it
aligned to its hugepage size.

struct ibv_mr *
mlx5_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp)
{
[...]
        /* Round start and end to page boundary if found in memory segments. */
        for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
                uintptr_t addr = (uintptr_t)ms[i].addr;
                size_t len = ms[i].len;
                unsigned int align = ms[i].hugepage_sz;

                if ((start > addr) && (start < addr + len))
                        start = RTE_ALIGN_FLOOR(start, align);
                if ((end > addr) && (end < addr + len))
                        end = RTE_ALIGN_CEIL(end, align);
        }

Thanks,
Yongseok

Re: [dpdk-dev] [PATCH v2 3/5] net/mlx5: use buffer address for LKEY search

[Nélio Laranjeiro]

On Mon, Jul 03, 2017 at 08:54:43PM +0000, Yongseok Koh wrote:
> 
> > On Jul 3, 2017, at 7:06 AM, Nélio Laranjeiro <nelio.laranjeiro@6wind.com> wrote:
> > 
> > On Fri, Jun 30, 2017 at 12:23:31PM -0700, Yongseok Koh wrote:
> >> When searching LKEY, if search key is mempool pointer, the 2nd cacheline
> >> has to be accessed and it even requires to check whether a buffer is
> >> indirect per every search. Instead, using address for search key can reduce
> >> cycles taken. And caching the last hit entry is beneficial as well.
> >> 
> >> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> >> ---
> >> drivers/net/mlx5/mlx5_mr.c   | 17 ++++++++++++++---
> >> drivers/net/mlx5/mlx5_rxtx.c | 39 +++++++++++++++++++++------------------
> >> drivers/net/mlx5/mlx5_rxtx.h |  4 +++-
> >> drivers/net/mlx5/mlx5_txq.c  |  3 +--
> >> 4 files changed, 39 insertions(+), 24 deletions(-)
> >> 
> >> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> >> index 0a3638460..287335179 100644
> >> --- a/drivers/net/mlx5/mlx5_mr.c
> >> +++ b/drivers/net/mlx5/mlx5_mr.c
> >> @@ -265,18 +266,28 @@ txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
> >> 	struct txq_mp2mr_mbuf_check_data data = {
> >> 		.ret = 0,
> >> 	};
> >> +	uintptr_t start;
> >> +	uintptr_t end;
> >> 	unsigned int i;
> >> 
> >> 	/* Register mempool only if the first element looks like a mbuf. */
> >> 	if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
> >> 			data.ret == -1)
> >> 		return;
> >> +	if (mlx5_check_mempool(mp, &start, &end) != 0) {
> >> +		ERROR("mempool %p: not virtually contiguous",
> >> +		      (void *)mp);
> >> +		return;
> >> +	}
> >> 	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
> >> -		if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) {
> >> +		struct ibv_mr *mr = txq_ctrl->txq.mp2mr[i].mr;
> >> +
> >> +		if (unlikely(mr == NULL)) {
> >> 			/* Unknown MP, add a new MR for it. */
> >> 			break;
> >> 		}
> >> -		if (txq_ctrl->txq.mp2mr[i].mp == mp)
> >> +		if (start >= (uintptr_t)mr->addr &&
> >> +		    end <= (uintptr_t)mr->addr + mr->length)
> >> 			return;
> >> 	}
> >> 	txq_mp2mr_reg(&txq_ctrl->txq, mp, i);
> > 
> > if (start >= (uintptr_t)mr->addr &&
> >     end <= (uintptr_t)mr->addr + mr->length)
> > 
> > Is this expected to have a memory region bigger than the memory pool
> > space?  I mean I was expecting to see strict equality in the addresses.
> In mlx5_mp2mr(), start/end of a memory region are rounded up to make it
> aligned to its hugepage size.
> 
> struct ibv_mr *
> mlx5_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp)
> {
> [...]
>         /* Round start and end to page boundary if found in memory segments. */
>         for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
>                 uintptr_t addr = (uintptr_t)ms[i].addr;
>                 size_t len = ms[i].len;
>                 unsigned int align = ms[i].hugepage_sz;
> 
>                 if ((start > addr) && (start < addr + len))
>                         start = RTE_ALIGN_FLOOR(start, align);
>                 if ((end > addr) && (end < addr + len))
>                         end = RTE_ALIGN_CEIL(end, align);
>         }
> 

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 4/5] net/mlx5: select Rx/Tx callbacks when starting device

[Yongseok Koh]

The callbacks are global to a device but the seletion is made every queue
configuration, which is redundant.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_rxq.c     | 2 --
 drivers/net/mlx5/mlx5_trigger.c | 3 +++
 drivers/net/mlx5/mlx5_txq.c     | 2 --
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 22be999e8..98b20eb92 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1226,8 +1226,6 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		DEBUG("%p: adding RX queue %p to list",
 		      (void *)dev, (void *)rxq_ctrl);
 		(*priv->rxqs)[idx] = &rxq_ctrl->rxq;
-		/* Update receive callback. */
-		priv_select_rx_function(priv);
 	}
 	priv_unlock(priv);
 	return -ret;
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 40f23da93..595a9e064 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -72,6 +72,9 @@ mlx5_dev_start(struct rte_eth_dev *dev)
 		priv_unlock(priv);
 		return 0;
 	}
+	/* Update Rx/Tx callback. */
+	priv_select_tx_function(priv);
+	priv_select_rx_function(priv);
 	DEBUG("%p: allocating and configuring hash RX queues", (void *)dev);
 	err = priv_create_hash_rxqs(priv);
 	if (!err)
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index ac9dfc5f0..03e23f58d 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -496,8 +496,6 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		DEBUG("%p: adding TX queue %p to list",
 		      (void *)dev, (void *)txq_ctrl);
 		(*priv->txqs)[idx] = &txq_ctrl->txq;
-		/* Update send callback. */
-		priv_select_tx_function(priv);
 	}
 	priv_unlock(priv);
 	return -ret;
-- 
2.11.0

Re: [dpdk-dev] [PATCH v2 4/5] net/mlx5: select Rx/Tx callbacks when starting device

[Nélio Laranjeiro]

On Fri, Jun 30, 2017 at 12:23:32PM -0700, Yongseok Koh wrote:
> The callbacks are global to a device but the seletion is made every queue
> configuration, which is redundant.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>

Same here, already acked in v1.

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v2 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Yongseok Koh]

To make vectorized burst routines enabled, it is required to run on x86_64
architecture which can support at least SSE4.1. If all the conditions are
met, the vectorized burst functions are enabled automatically. The decision
is made individually on RX and TX. There's no PMD option to make a
selection.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/Makefile            |   10 +
 drivers/net/mlx5/mlx5_defs.h         |   18 +
 drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
 drivers/net/mlx5/mlx5_rxq.c          |   55 +-
 drivers/net/mlx5/mlx5_rxtx.c         |  339 ++------
 drivers/net/mlx5/mlx5_rxtx.h         |  283 ++++++-
 drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_txq.c          |    2 +-
 8 files changed, 1909 insertions(+), 277 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 8b8f6eaac..b0f8560a2 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -39,6 +39,11 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c
+ifneq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),)
+ifeq ($(CONFIG_RTE_ARCH_X86_64),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx_vec_sse.c
+endif
+endif
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_trigger.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mac.c
@@ -85,6 +90,11 @@ ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
 CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
 endif
 
+#check if flag for SSE4.1 is already on, if not set it up manually
+ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),)
+CFLAGS_mlx5_rxtx_vec_sse.o += -msse4.1
+endif
+
 include $(RTE_SDK)/mk/rte.lib.mk
 
 # Generate and clean-up mlx5_autoconf.h.
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 201bb3362..a76bc6f65 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -89,4 +89,22 @@
 /* Maximum Packet headers size (L2+L3+L4) for TSO. */
 #define MLX5_MAX_TSO_HEADER 128
 
+/* Default minimum number of Tx queues for vectorized Tx. */
+#define MLX5_VPMD_MIN_TXQS 4
+
+/* Threshold of buffer replenishment for vectorized Rx. */
+#define MLX5_VPMD_RXQ_RPLNSH_THRESH   64U
+
+/* Maximum size of burst for vectorized Rx. */
+#define MLX5_VPMD_RX_MAX_BURST        MLX5_VPMD_RXQ_RPLNSH_THRESH
+
+/*
+ * Maximum size of burst for vectorized Tx. This is related to the maximum size
+ * of Enhaned MPW (eMPW) WQE as vectorized Tx is supported with eMPW.
+ */
+#define MLX5_VPMD_TX_MAX_BURST        32U
+
+/* Number of packets vectorized Rx can simultaneously process in a loop. */
+#define MLX5_VPMD_DESCS_PER_LOOP      4
+
 #endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index eadf452b9..7f3cef547 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -723,7 +723,8 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 
 	};
 
-	if (dev->rx_pkt_burst == mlx5_rx_burst)
+	if (dev->rx_pkt_burst == mlx5_rx_burst ||
+	    dev->rx_pkt_burst == mlx5_rx_burst_vec)
 		return ptypes;
 	return NULL;
 }
@@ -924,7 +925,7 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	int ret = 0;
 	unsigned int i;
 	uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) =
-		mlx5_rx_burst;
+		dev->rx_pkt_burst;
 	unsigned int max_frame_len;
 	int rehash;
 	int restart = priv->started;
@@ -944,7 +945,7 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	/* Temporarily replace RX handler with a fake one, assuming it has not
 	 * been copied elsewhere. */
 	dev->rx_pkt_burst = removed_rx_burst;
-	/* Make sure everyone has left mlx5_rx_burst() and uses
+	/* Make sure everyone has left dev->rx_pkt_burst() and uses
 	 * removed_rx_burst() instead. */
 	rte_wmb();
 	usleep(1000);
@@ -1584,9 +1585,16 @@ priv_select_tx_function(struct priv *priv)
 	priv->dev->tx_pkt_burst = mlx5_tx_burst;
 	/* Select appropriate TX function. */
 	if (priv->mps == MLX5_MPW_ENHANCED) {
-		priv->dev->tx_pkt_burst =
-			mlx5_tx_burst_empw;
-		DEBUG("selected Enhanced MPW TX function");
+		if (priv_check_vec_tx_support(priv) > 0) {
+			if (priv_check_raw_vec_tx_support(priv) > 0)
+				priv->dev->tx_pkt_burst = mlx5_tx_burst_raw_vec;
+			else
+				priv->dev->tx_pkt_burst = mlx5_tx_burst_vec;
+			DEBUG("selected Enhanced MPW TX vectorized function");
+		} else {
+			priv->dev->tx_pkt_burst = mlx5_tx_burst_empw;
+			DEBUG("selected Enhanced MPW TX function");
+		}
 	} else if (priv->mps && priv->txq_inline) {
 		priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline;
 		DEBUG("selected MPW inline TX function");
@@ -1605,5 +1613,11 @@ priv_select_tx_function(struct priv *priv)
 void
 priv_select_rx_function(struct priv *priv)
 {
-	priv->dev->rx_pkt_burst = mlx5_rx_burst;
+	if (priv_check_vec_rx_support(priv) > 0) {
+		priv_prep_vec_rx_function(priv);
+		priv->dev->rx_pkt_burst = mlx5_rx_burst_vec;
+		DEBUG("selected RX vectorized function");
+	} else {
+		priv->dev->rx_pkt_burst = mlx5_rx_burst;
+	}
 }
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 98b20eb92..4842cda92 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -634,6 +634,41 @@ priv_rehash_flows(struct priv *priv)
 }
 
 /**
+ * Unlike regular Rx function, vPMD Rx doesn't replace mbufs immediately when
+ * receiving packets. Instead it replaces later in bulk. In rxq->elts[], entries
+ * from rq_pi to rq_ci are owned by device but the rest is already delivered to
+ * application. In order not to reuse those mbufs by rxq_alloc_elts(), this
+ * function must be called to replace used mbufs.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+static int
+rxq_trim_elts(struct rxq *rxq)
+{
+	const uint16_t q_n = (1 << rxq->elts_n);
+	const uint16_t q_mask = q_n - 1;
+	uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi);
+	uint16_t i;
+
+	if (!rxq->trim_elts)
+		return 0;
+	for (i = 0; i < used; ++i) {
+		struct rte_mbuf *buf;
+		buf = rte_pktmbuf_alloc(rxq->mp);
+		if (!buf)
+			return ENOMEM;
+		(*rxq->elts)[(rxq->rq_ci + i) & q_mask] = buf;
+	}
+	rxq->rq_pi = rxq->rq_ci;
+	rxq->trim_elts = 0;
+	return 0;
+}
+
+/**
  * Allocate RX queue elements.
  *
  * @param rxq_ctrl
@@ -800,6 +835,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl)
 		return err;
 	}
 	/* Snatch mbufs from original queue. */
+	claim_zero(rxq_trim_elts(&rxq_ctrl->rxq));
 	claim_zero(rxq_alloc_elts(rxq_ctrl, elts_n, rxq_ctrl->rxq.elts));
 	for (i = 0; i != elts_n; ++i) {
 		struct rte_mbuf *buf = (*rxq_ctrl->rxq.elts)[i];
@@ -860,6 +896,7 @@ rxq_setup(struct rxq_ctrl *tmpl)
 	tmpl->rxq.cqe_n = log2above(cq_info.cqe_cnt);
 	tmpl->rxq.cq_ci = 0;
 	tmpl->rxq.rq_ci = 0;
+	tmpl->rxq.rq_pi = 0;
 	tmpl->rxq.cq_db = cq_info.dbrec;
 	tmpl->rxq.wqes =
 		(volatile struct mlx5_wqe_data_seg (*)[])
@@ -993,7 +1030,12 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
 	if (priv->cqe_comp) {
 		attr.cq.comp_mask |= IBV_EXP_CQ_INIT_ATTR_FLAGS;
 		attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE;
-		cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
+		/*
+		 * For vectorized Rx, it must not be doubled in order to
+		 * make cq_ci and rq_ci aligned.
+		 */
+		if (rxq_check_vec_rx_support(&tmpl.rxq) < 0)
+			cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
 	}
 	tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, tmpl.channel, 0,
 				    &attr.cq);
@@ -1103,7 +1145,9 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
 	if (rxq_ctrl->rxq.elts_n) {
 		assert(1 << rxq_ctrl->rxq.elts_n == desc);
 		assert(rxq_ctrl->rxq.elts != tmpl.rxq.elts);
-		ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts);
+		ret = rxq_trim_elts(&rxq_ctrl->rxq);
+		if (!ret)
+			ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts);
 	} else
 		ret = rxq_alloc_elts(&tmpl, desc, NULL);
 	if (ret) {
@@ -1165,6 +1209,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	struct priv *priv = dev->data->dev_private;
 	struct rxq *rxq = (*priv->rxqs)[idx];
 	struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
+	const uint16_t desc_pad = MLX5_VPMD_DESCS_PER_LOOP; /* For vPMD. */
 	int ret;
 
 	if (mlx5_is_secondary())
@@ -1198,7 +1243,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		if (rxq_ctrl->rxq.elts_n != log2above(desc)) {
 			rxq_ctrl = rte_realloc(rxq_ctrl,
 					       sizeof(*rxq_ctrl) +
-					       desc * sizeof(struct rte_mbuf *),
+					       (desc + desc_pad) *
+						sizeof(struct rte_mbuf *),
 					       RTE_CACHE_LINE_SIZE);
 			if (!rxq_ctrl) {
 				ERROR("%p: unable to reallocate queue index %u",
@@ -1209,7 +1255,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		}
 	} else {
 		rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl) +
-					     desc * sizeof(struct rte_mbuf *),
+					     (desc + desc_pad) *
+					      sizeof(struct rte_mbuf *),
 					     0, socket);
 		if (rxq_ctrl == NULL) {
 			ERROR("%p: unable to allocate queue index %u",
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 688ee9028..16bac5563 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -92,102 +92,6 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe,
 static __rte_always_inline uint32_t
 rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe);
 
-#ifndef NDEBUG
-
-/**
- * Verify or set magic value in CQE.
- *
- * @param cqe
- *   Pointer to CQE.
- *
- * @return
- *   0 the first time.
- */
-static inline int
-check_cqe_seen(volatile struct mlx5_cqe *cqe)
-{
-	static const uint8_t magic[] = "seen";
-	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
-	int ret = 1;
-	unsigned int i;
-
-	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
-		if (!ret || (*buf)[i] != magic[i]) {
-			ret = 0;
-			(*buf)[i] = magic[i];
-		}
-	return ret;
-}
-
-#endif /* NDEBUG */
-
-/**
- * Check whether CQE is valid.
- *
- * @param cqe
- *   Pointer to CQE.
- * @param cqes_n
- *   Size of completion queue.
- * @param ci
- *   Consumer index.
- *
- * @return
- *   0 on success, 1 on failure.
- */
-static inline int
-check_cqe(volatile struct mlx5_cqe *cqe,
-	  unsigned int cqes_n, const uint16_t ci)
-{
-	uint16_t idx = ci & cqes_n;
-	uint8_t op_own = cqe->op_own;
-	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
-	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
-
-	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
-		return 1; /* No CQE. */
-#ifndef NDEBUG
-	if ((op_code == MLX5_CQE_RESP_ERR) ||
-	    (op_code == MLX5_CQE_REQ_ERR)) {
-		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
-		uint8_t syndrome = err_cqe->syndrome;
-
-		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
-		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
-			return 0;
-		if (!check_cqe_seen(cqe))
-			ERROR("unexpected CQE error %u (0x%02x)"
-			      " syndrome 0x%02x",
-			      op_code, op_code, syndrome);
-		return 1;
-	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
-		   (op_code != MLX5_CQE_REQ)) {
-		if (!check_cqe_seen(cqe))
-			ERROR("unexpected CQE opcode %u (0x%02x)",
-			      op_code, op_code);
-		return 1;
-	}
-#endif /* NDEBUG */
-	return 0;
-}
-
-/**
- * Return the address of the WQE.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param  wqe_ci
- *   WQE consumer index.
- *
- * @return
- *   WQE address.
- */
-static inline uintptr_t *
-tx_mlx5_wqe(struct txq *txq, uint16_t ci)
-{
-	ci &= ((1 << txq->wqe_n) - 1);
-	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
-}
-
 /**
  * Return the size of tailroom of WQ.
  *
@@ -245,174 +149,6 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
 }
 
 /**
- * Manage TX completions.
- *
- * When sending a burst, mlx5_tx_burst() posts several WRs.
- *
- * @param txq
- *   Pointer to TX queue structure.
- */
-static inline void
-txq_complete(struct txq *txq)
-{
-	const uint16_t elts_n = 1 << txq->elts_n;
-	const uint16_t elts_m = elts_n - 1;
-	const unsigned int cqe_n = 1 << txq->cqe_n;
-	const unsigned int cqe_cnt = cqe_n - 1;
-	uint16_t elts_free = txq->elts_tail;
-	uint16_t elts_tail;
-	uint16_t cq_ci = txq->cq_ci;
-	volatile struct mlx5_cqe *cqe = NULL;
-	volatile struct mlx5_wqe_ctrl *ctrl;
-	struct rte_mbuf *m, *free[elts_n];
-	struct rte_mempool *pool = NULL;
-	unsigned int blk_n = 0;
-
-	do {
-		volatile struct mlx5_cqe *tmp;
-
-		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
-		if (check_cqe(tmp, cqe_n, cq_ci))
-			break;
-		cqe = tmp;
-#ifndef NDEBUG
-		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
-			if (!check_cqe_seen(cqe))
-				ERROR("unexpected compressed CQE, TX stopped");
-			return;
-		}
-		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
-		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
-			if (!check_cqe_seen(cqe))
-				ERROR("unexpected error CQE, TX stopped");
-			return;
-		}
-#endif /* NDEBUG */
-		++cq_ci;
-	} while (1);
-	if (unlikely(cqe == NULL))
-		return;
-	txq->wqe_pi = ntohs(cqe->wqe_counter);
-	ctrl = (volatile struct mlx5_wqe_ctrl *)
-		tx_mlx5_wqe(txq, txq->wqe_pi);
-	elts_tail = ctrl->ctrl3;
-	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
-	/* Free buffers. */
-	while (elts_free != elts_tail) {
-		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
-		if (likely(m != NULL)) {
-			if (likely(m->pool == pool)) {
-				free[blk_n++] = m;
-			} else {
-				if (likely(pool != NULL))
-					rte_mempool_put_bulk(pool,
-							     (void *)free,
-							     blk_n);
-				free[0] = m;
-				pool = m->pool;
-				blk_n = 1;
-			}
-		}
-	}
-	if (blk_n)
-		rte_mempool_put_bulk(pool, (void *)free, blk_n);
-#ifndef NDEBUG
-	elts_free = txq->elts_tail;
-	/* Poisoning. */
-	while (elts_free != elts_tail) {
-		memset(&(*txq->elts)[elts_free & elts_m],
-		       0x66,
-		       sizeof((*txq->elts)[elts_free & elts_m]));
-		++elts_free;
-	}
-#endif
-	txq->cq_ci = cq_ci;
-	txq->elts_tail = elts_tail;
-	/* Update the consumer index. */
-	rte_wmb();
-	*txq->cq_db = htonl(cq_ci);
-}
-
-/**
- * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
- * the cloned mbuf is allocated is returned instead.
- *
- * @param buf
- *   Pointer to mbuf.
- *
- * @return
- *   Memory pool where data is located for given mbuf.
- */
-static struct rte_mempool *
-txq_mb2mp(struct rte_mbuf *buf)
-{
-	if (unlikely(RTE_MBUF_INDIRECT(buf)))
-		return rte_mbuf_from_indirect(buf)->pool;
-	return buf->pool;
-}
-
-/**
- * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
- * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
- * remove an entry first.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param[in] mp
- *   Memory Pool for which a Memory Region lkey must be returned.
- *
- * @return
- *   mr->lkey on success, (uint32_t)-1 on failure.
- */
-static inline uint32_t
-txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
-{
-	uint16_t i = txq->mr_cache_idx;
-	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
-
-	assert(i < RTE_DIM(txq->mp2mr));
-	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
-		return txq->mp2mr[i].lkey;
-	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
-		if (unlikely(txq->mp2mr[i].mr == NULL)) {
-			/* Unknown MP, add a new MR for it. */
-			break;
-		}
-		if (txq->mp2mr[i].start <= addr &&
-		    txq->mp2mr[i].end >= addr) {
-			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
-			assert(htonl(txq->mp2mr[i].mr->lkey) ==
-			       txq->mp2mr[i].lkey);
-			txq->mr_cache_idx = i;
-			return txq->mp2mr[i].lkey;
-		}
-	}
-	txq->mr_cache_idx = 0;
-	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
-}
-
-/**
- * Ring TX queue doorbell.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param wqe
- *   Pointer to the last WQE posted in the NIC.
- */
-static inline void
-mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
-{
-	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
-	volatile uint64_t *src = ((volatile uint64_t *)wqe);
-
-	rte_wmb();
-	*txq->qp_db = htonl(txq->wqe_ci);
-	/* Ensure ordering between DB record and BF copy. */
-	rte_wmb();
-	*dst = *src;
-}
-
-/**
  * DPDK callback to check the status of a tx descriptor.
  *
  * @param tx_queue
@@ -2138,3 +1874,78 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	(void)pkts_n;
 	return 0;
 }
+
+/*
+ * Vectorized Rx/Tx routines are not compiled in when required vector
+ * instructions are not supported on a target architecture. The following null
+ * stubs are needed for linkage when those are not included outside of this file
+ * (e.g.  mlx5_rxtx_vec_sse.c for x86).
+ */
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_txq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_txq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_rxq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+int __attribute__((weak))
+txq_check_vec_tx_support(struct txq *txq)
+{
+	(void)txq;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_raw_vec_tx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_vec_tx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+rxq_check_vec_rx_support(struct rxq *rxq)
+{
+	(void)rxq;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_vec_rx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+void __attribute__((weak))
+priv_prep_vec_rx_function(struct priv *priv)
+{
+	(void)priv;
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 51e258a15..2d0894fcd 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -115,10 +115,13 @@ struct rxq {
 	unsigned int port_id:8;
 	unsigned int rss_hash:1; /* RSS hash result is enabled. */
 	unsigned int mark:1; /* Marked flow available on the queue. */
-	unsigned int :8; /* Remaining bits. */
+	unsigned int pending_err:1; /* CQE error needs to be handled. */
+	unsigned int trim_elts:1; /* Whether elts needs clean-up. */
+	unsigned int :6; /* Remaining bits. */
 	volatile uint32_t *rq_db;
 	volatile uint32_t *cq_db;
 	uint16_t rq_ci;
+	uint16_t rq_pi;
 	uint16_t cq_ci;
 	volatile struct mlx5_wqe_data_seg(*wqes)[];
 	volatile struct mlx5_cqe(*cqes)[];
@@ -126,6 +129,8 @@ struct rxq {
 	struct rte_mbuf *(*elts)[];
 	struct rte_mempool *mp;
 	struct mlx5_rxq_stats stats;
+	uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */
+	struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
 } __rte_cache_aligned;
 
 /* RX queue control descriptor. */
@@ -261,6 +266,7 @@ struct txq {
 	uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */
 	uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
 	uint32_t qp_num_8s; /* QP number shifted by 8. */
+	uint32_t flags; /* Flags for Tx Queue. */
 	volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
 	volatile void *wqes; /* Work queue (use volatile to write into). */
 	volatile uint32_t *qp_db; /* Work queue doorbell. */
@@ -338,10 +344,285 @@ uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t);
 int mlx5_rx_descriptor_status(void *, uint16_t);
 int mlx5_tx_descriptor_status(void *, uint16_t);
 
+/* Vectorized version of mlx5_rxtx.c */
+int txq_check_vec_tx_support(struct txq *);
+int priv_check_raw_vec_tx_support(struct priv *);
+int priv_check_vec_tx_support(struct priv *);
+int rxq_check_vec_rx_support(struct rxq *);
+int priv_check_vec_rx_support(struct priv *);
+void priv_prep_vec_rx_function(struct priv *);
+uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t);
+uint16_t mlx5_tx_burst_vec(void *, struct rte_mbuf **, uint16_t);
+uint16_t mlx5_rx_burst_vec(void *, struct rte_mbuf **, uint16_t);
+
 /* mlx5_mr.c */
 
 struct ibv_mr *mlx5_mp2mr(struct ibv_pd *, struct rte_mempool *);
 void txq_mp2mr_iter(struct rte_mempool *, void *);
 uint32_t txq_mp2mr_reg(struct txq *, struct rte_mempool *, unsigned int);
 
+#ifndef NDEBUG
+
+/**
+ * Verify or set magic value in CQE.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ *
+ * @return
+ *   0 the first time.
+ */
+static inline int
+check_cqe_seen(volatile struct mlx5_cqe *cqe)
+{
+	static const uint8_t magic[] = "seen";
+	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
+	int ret = 1;
+	unsigned int i;
+
+	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
+		if (!ret || (*buf)[i] != magic[i]) {
+			ret = 0;
+			(*buf)[i] = magic[i];
+		}
+	return ret;
+}
+
+#endif /* NDEBUG */
+
+/**
+ * Check whether CQE is valid.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ * @param cqes_n
+ *   Size of completion queue.
+ * @param ci
+ *   Consumer index.
+ *
+ * @return
+ *   0 on success, 1 on failure.
+ */
+static inline int
+check_cqe(volatile struct mlx5_cqe *cqe,
+	  unsigned int cqes_n, const uint16_t ci)
+{
+	uint16_t idx = ci & cqes_n;
+	uint8_t op_own = cqe->op_own;
+	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
+	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
+
+	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
+		return 1; /* No CQE. */
+#ifndef NDEBUG
+	if ((op_code == MLX5_CQE_RESP_ERR) ||
+	    (op_code == MLX5_CQE_REQ_ERR)) {
+		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
+		uint8_t syndrome = err_cqe->syndrome;
+
+		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
+		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
+			return 0;
+		if (!check_cqe_seen(cqe))
+			ERROR("unexpected CQE error %u (0x%02x)"
+			      " syndrome 0x%02x",
+			      op_code, op_code, syndrome);
+		return 1;
+	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
+		   (op_code != MLX5_CQE_REQ)) {
+		if (!check_cqe_seen(cqe))
+			ERROR("unexpected CQE opcode %u (0x%02x)",
+			      op_code, op_code);
+		return 1;
+	}
+#endif /* NDEBUG */
+	return 0;
+}
+
+/**
+ * Return the address of the WQE.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param  wqe_ci
+ *   WQE consumer index.
+ *
+ * @return
+ *   WQE address.
+ */
+static inline uintptr_t *
+tx_mlx5_wqe(struct txq *txq, uint16_t ci)
+{
+	ci &= ((1 << txq->wqe_n) - 1);
+	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
+}
+
+/**
+ * Manage TX completions.
+ *
+ * When sending a burst, mlx5_tx_burst() posts several WRs.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ */
+static inline void
+txq_complete(struct txq *txq)
+{
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int cqe_n = 1 << txq->cqe_n;
+	const unsigned int cqe_cnt = cqe_n - 1;
+	uint16_t elts_free = txq->elts_tail;
+	uint16_t elts_tail;
+	uint16_t cq_ci = txq->cq_ci;
+	volatile struct mlx5_cqe *cqe = NULL;
+	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	struct rte_mempool *pool = NULL;
+	unsigned int blk_n = 0;
+
+	do {
+		volatile struct mlx5_cqe *tmp;
+
+		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
+		if (check_cqe(tmp, cqe_n, cq_ci))
+			break;
+		cqe = tmp;
+#ifndef NDEBUG
+		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
+			if (!check_cqe_seen(cqe))
+				ERROR("unexpected compressed CQE, TX stopped");
+			return;
+		}
+		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
+		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
+			if (!check_cqe_seen(cqe))
+				ERROR("unexpected error CQE, TX stopped");
+			return;
+		}
+#endif /* NDEBUG */
+		++cq_ci;
+	} while (1);
+	if (unlikely(cqe == NULL))
+		return;
+	txq->wqe_pi = ntohs(cqe->wqe_counter);
+	ctrl = (volatile struct mlx5_wqe_ctrl *)
+		tx_mlx5_wqe(txq, txq->wqe_pi);
+	elts_tail = ctrl->ctrl3;
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
+	/* Free buffers. */
+	while (elts_free != elts_tail) {
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (likely(m->pool == pool)) {
+				free[blk_n++] = m;
+			} else {
+				if (likely(pool != NULL))
+					rte_mempool_put_bulk(pool,
+							     (void *)free,
+							     blk_n);
+				free[0] = m;
+				pool = m->pool;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(pool, (void *)free, blk_n);
+#ifndef NDEBUG
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
+		memset(&(*txq->elts)[elts_free & elts_m],
+		       0x66,
+		       sizeof((*txq->elts)[elts_free & elts_m]));
+		++elts_free;
+	}
+#endif
+	txq->cq_ci = cq_ci;
+	txq->elts_tail = elts_tail;
+	/* Update the consumer index. */
+	rte_wmb();
+	*txq->cq_db = htonl(cq_ci);
+}
+
+/**
+ * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
+ * the cloned mbuf is allocated is returned instead.
+ *
+ * @param buf
+ *   Pointer to mbuf.
+ *
+ * @return
+ *   Memory pool where data is located for given mbuf.
+ */
+static struct rte_mempool *
+txq_mb2mp(struct rte_mbuf *buf)
+{
+	if (unlikely(RTE_MBUF_INDIRECT(buf)))
+		return rte_mbuf_from_indirect(buf)->pool;
+	return buf->pool;
+}
+
+/**
+ * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
+ * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
+ * remove an entry first.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param[in] mp
+ *   Memory Pool for which a Memory Region lkey must be returned.
+ *
+ * @return
+ *   mr->lkey on success, (uint32_t)-1 on failure.
+ */
+static inline uint32_t
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
+{
+	uint16_t i = txq->mr_cache_idx;
+	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
+
+	assert(i < RTE_DIM(txq->mp2mr));
+	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
+		return txq->mp2mr[i].lkey;
+	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
+		if (unlikely(txq->mp2mr[i].mr == NULL)) {
+			/* Unknown MP, add a new MR for it. */
+			break;
+		}
+		if (txq->mp2mr[i].start <= addr &&
+		    txq->mp2mr[i].end >= addr) {
+			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
+			assert(htonl(txq->mp2mr[i].mr->lkey) ==
+			       txq->mp2mr[i].lkey);
+			txq->mr_cache_idx = i;
+			return txq->mp2mr[i].lkey;
+		}
+	}
+	txq->mr_cache_idx = 0;
+	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
+}
+
+/**
+ * Ring TX queue doorbell.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param wqe
+ *   Pointer to the last WQE posted in the NIC.
+ */
+static inline void
+mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
+{
+	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
+	volatile uint64_t *src = ((volatile uint64_t *)wqe);
+
+	rte_wmb();
+	*txq->qp_db = htonl(txq->wqe_ci);
+	/* Ensure ordering between DB record and BF copy. */
+	rte_wmb();
+	*dst = *src;
+}
+
 #endif /* RTE_PMD_MLX5_RXTX_H_ */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.c b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
new file mode 100644
index 000000000..566a60635
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
@@ -0,0 +1,1451 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5_hw.h>
+#include <infiniband/arch.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+/* DPDK headers don't like -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+#include <smmintrin.h>
+
+#ifndef __INTEL_COMPILER
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#endif
+
+/**
+ * Fill in DSEGs in Tx descriptors (WQE).
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param dseg
+ *   Pointer to DSEG to be writen.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param n
+ *   Number of packets to be filled.
+ */
+static inline void
+txq_wr_dseg_v(struct txq *txq, __m128i *dseg,
+	      struct rte_mbuf **pkts, unsigned int n)
+{
+	unsigned int pos;
+	uintptr_t addr;
+	const __m128i shuf_mask_dseg =
+		_mm_set_epi8(8,  9, 10, 11, /* addr, bswap64 */
+			    12, 13, 14, 15,
+			     7,  6,  5,  4, /* lkey */
+			     0,  1,  2,  3  /* length, bswap32 */);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t tx_byte = 0;
+#endif
+
+	for (pos = 0; pos < n; ++pos, ++dseg) {
+		__m128i desc;
+		struct rte_mbuf *pkt = pkts[pos];
+
+		addr = rte_pktmbuf_mtod(pkt, uintptr_t);
+		desc = _mm_set_epi32(addr >> 32,
+				     addr,
+				     txq_mb2mr(txq, pkt),
+				     DATA_LEN(pkt));
+		desc = _mm_shuffle_epi8(desc, shuf_mask_dseg);
+		_mm_store_si128(dseg, desc);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		tx_byte += DATA_LEN(pkt);
+#endif
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.obytes += tx_byte;
+#endif
+}
+
+/**
+ * Count the number of continuous single segment packets. The first packet must
+ * be a single segment packet.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ *
+ * @return
+ *   Number of continuous single segment packets.
+ */
+static inline unsigned int
+txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	unsigned int pos;
+
+	if (!pkts_n)
+		return 0;
+	assert(NB_SEGS(pkts[0]) == 1);
+	/* Count the number of continuous single segment packets. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if (NB_SEGS(pkts[pos]) > 1)
+			break;
+	return pos;
+}
+
+/**
+ * Count the number of packets having same ol_flags and calculate cs_flags.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ * @param cs_flags
+ *   Pointer of flags to be returned.
+ *
+ * @return
+ *   Number of packets having same ol_flags.
+ */
+static inline unsigned int
+txq_calc_offload(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+		 uint8_t *cs_flags)
+{
+	unsigned int pos;
+	const uint64_t ol_mask =
+		PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
+		PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
+		PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
+
+	if (!pkts_n)
+		return 0;
+	/* Count the number of packets having same ol_flags. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
+			break;
+	/* Should open another MPW session for the rest. */
+	if (pkts[0]->ol_flags &
+	    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+		const uint64_t is_tunneled =
+			pkts[0]->ol_flags &
+			(PKT_TX_TUNNEL_GRE |
+			 PKT_TX_TUNNEL_VXLAN);
+
+		if (is_tunneled && txq->tunnel_en) {
+			*cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
+				    MLX5_ETH_WQE_L4_INNER_CSUM;
+			if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+				*cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+		} else {
+			*cs_flags = MLX5_ETH_WQE_L3_CSUM |
+				    MLX5_ETH_WQE_L4_CSUM;
+		}
+	}
+	return pos;
+}
+
+/**
+ * Send multi-segmented packets until it encounters a single segment packet in
+ * the pkts list.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static uint16_t
+txq_scatter_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n;
+	volatile struct mlx5_wqe *wqe = NULL;
+
+	assert(elts_n > pkts_n);
+	txq_complete(txq);
+	/* A CQE slot must always be available. */
+	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
+	if (unlikely(!pkts_n))
+		return 0;
+	for (n = 0; n < pkts_n; ++n) {
+		struct rte_mbuf *buf = pkts[n];
+		unsigned int segs_n = buf->nb_segs;
+		unsigned int ds = nb_dword_in_hdr;
+		unsigned int len = PKT_LEN(buf);
+		uint16_t wqe_ci = txq->wqe_ci;
+		const __m128i shuf_mask_ctrl =
+			_mm_set_epi8(15, 14, 13, 12,
+				      8,  9, 10, 11, /* bswap32 */
+				      4,  5,  6,  7, /* bswap32 */
+				      0,  1,  2,  3  /* bswap32 */);
+		uint8_t cs_flags = 0;
+		uint16_t max_elts;
+		uint16_t max_wqe;
+		__m128i *t_wqe, *dseg;
+		__m128i ctrl;
+
+		assert(segs_n);
+		max_elts = elts_n - (elts_head - txq->elts_tail);
+		max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi);
+		/*
+		 * A MPW session consumes 2 WQEs at most to
+		 * include MLX5_MPW_DSEG_MAX pointers.
+		 */
+		if (segs_n == 1 ||
+		    max_elts < segs_n || max_wqe < 2)
+			break;
+		wqe = &((volatile struct mlx5_wqe64 *)
+			 txq->wqes)[wqe_ci & wq_mask].hdr;
+		if (buf->ol_flags &
+		     (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+			const uint64_t is_tunneled = buf->ol_flags &
+						      (PKT_TX_TUNNEL_GRE |
+						       PKT_TX_TUNNEL_VXLAN);
+
+			if (is_tunneled && txq->tunnel_en) {
+				cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
+					   MLX5_ETH_WQE_L4_INNER_CSUM;
+				if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+					cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+			} else {
+				cs_flags = MLX5_ETH_WQE_L3_CSUM |
+					   MLX5_ETH_WQE_L4_CSUM;
+			}
+		}
+		/* Title WQEBB pointer. */
+		t_wqe = (__m128i *)wqe;
+		dseg = (__m128i *)(wqe + 1);
+		do {
+			if (!(ds++ % nb_dword_per_wqebb)) {
+				dseg = (__m128i *)
+					&((volatile struct mlx5_wqe64 *)
+					   txq->wqes)[++wqe_ci & wq_mask];
+			}
+			txq_wr_dseg_v(txq, dseg++, &buf, 1);
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			buf = buf->next;
+		} while (--segs_n);
+		if (ds % nb_dword_per_wqebb)
+			++wqe_ci;
+		/* Fill CTRL in the header. */
+		ctrl = _mm_set_epi32(0, 0, txq->qp_num_8s | ds,
+				     MLX5_OPC_MOD_MPW << 24 |
+				     txq->wqe_ci << 8 | MLX5_OPCODE_TSO);
+		ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+		_mm_store_si128(t_wqe, ctrl);
+		/* Fill ESEG in the header. */
+		_mm_store_si128(t_wqe + 1,
+				_mm_set_epi16(0, 0, 0, 0,
+					      htons(len), cs_flags,
+					      0, 0));
+		txq->wqe_ci = wqe_ci;
+	}
+	if (!n)
+		return 0;
+	txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
+	txq->elts_head = elts_head;
+	if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+		wqe->ctrl[2] = htonl(8);
+		wqe->ctrl[3] = txq->elts_head;
+		txq->elts_comp = 0;
+		++txq->cq_pi;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += n;
+#endif
+	mlx5_tx_dbrec(txq, wqe);
+	return n;
+}
+
+/**
+ * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
+ * it returns to make it processed by txq_scatter_v(). All the packets in
+ * the pkts list should be single segment packets having same offload flags.
+ * This must be checked by txq_check_multiseg() and txq_calc_offload().
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent.
+ * @param cs_flags
+ *   Checksum offload flags to be written in the descriptor.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static inline uint16_t
+txq_burst_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+	    uint8_t cs_flags)
+{
+	struct rte_mbuf **elts;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n = 0;
+	unsigned int pos;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	uint32_t comp_req = 0;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	uint16_t wq_idx = txq->wqe_ci & wq_mask;
+	volatile struct mlx5_wqe64 *wq =
+		&((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx];
+	volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq;
+	const __m128i shuf_mask_ctrl =
+		_mm_set_epi8(15, 14, 13, 12,
+			      8,  9, 10, 11, /* bswap32 */
+			      4,  5,  6,  7, /* bswap32 */
+			      0,  1,  2,  3  /* bswap32 */);
+	__m128i *t_wqe, *dseg;
+	__m128i ctrl;
+
+	/* Make sure all packets can fit into a single WQE. */
+	assert(pkts_n <= MLX5_VPMD_TX_MAX_BURST);
+	assert(elts_n > pkts_n);
+	txq_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	/* A CQE slot must always be available. */
+	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
+	if (unlikely(!pkts_n))
+		return 0;
+	elts = &(*txq->elts)[elts_head & elts_m];
+	/* Loop for available tailroom first. */
+	n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n);
+	for (pos = 0; pos < (n & -2); pos += 2)
+		_mm_storeu_si128((__m128i *)&elts[pos],
+				 _mm_loadu_si128((__m128i *)&pkts[pos]));
+	if (n & 1)
+		elts[pos] = pkts[pos];
+	/* Check if it crosses the end of the queue. */
+	if (unlikely(n < pkts_n)) {
+		elts = &(*txq->elts)[0];
+		for (pos = 0; pos < pkts_n - n; ++pos)
+			elts[pos] = pkts[n + pos];
+	}
+	txq->elts_head += pkts_n;
+	/* Save title WQEBB pointer. */
+	t_wqe = (__m128i *)wqe;
+	dseg = (__m128i *)(wqe + 1);
+	/* Calculate the number of entries to the end. */
+	n = RTE_MIN(
+		(wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr,
+		pkts_n);
+	/* Fill DSEGs. */
+	txq_wr_dseg_v(txq, dseg, pkts, n);
+	/* Check if it crosses the end of the queue. */
+	if (n < pkts_n) {
+		dseg = (__m128i *)txq->wqes;
+		txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n);
+	}
+	if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
+		txq->elts_comp += pkts_n;
+	} else {
+		/* Request a completion. */
+		txq->elts_comp = 0;
+		++txq->cq_pi;
+		comp_req = 8;
+	}
+	/* Fill CTRL in the header. */
+	ctrl = _mm_set_epi32(txq->elts_head, comp_req,
+			     txq->qp_num_8s | (pkts_n + 2),
+			     MLX5_OPC_MOD_ENHANCED_MPSW << 24 |
+				txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW);
+	ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+	_mm_store_si128(t_wqe, ctrl);
+	/* Fill ESEG in the header. */
+	_mm_store_si128(t_wqe + 1,
+			_mm_set_epi8(0, 0, 0, 0,
+				     0, 0, 0, 0,
+				     0, 0, 0, cs_flags,
+				     0, 0, 0, 0));
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += pkts_n;
+#endif
+	txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) /
+		       nb_dword_per_wqebb;
+	/* Ring QP doorbell. */
+	mlx5_tx_dbrec(txq, wqe);
+	return pkts_n;
+}
+
+/**
+ * DPDK callback for vectorized TX.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
+		      uint16_t pkts_n)
+{
+	struct txq *txq = (struct txq *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint16_t n;
+		uint16_t ret;
+
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, 0);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * DPDK callback for vectorized TX with multi-seg packets and offload.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct txq *txq = (struct txq *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint8_t cs_flags = 0;
+		uint16_t n;
+		uint16_t ret;
+
+		/* Transmit multi-seg packets in the head of pkts list. */
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) &&
+		    NB_SEGS(pkts[nb_tx]) > 1)
+			nb_tx += txq_scatter_v(
+					txq, &pkts[nb_tx], pkts_n - nb_tx);
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS))
+			n = txq_check_multiseg(&pkts[nb_tx], n);
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
+			n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * Store free buffers to RX SW ring.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be stored.
+ * @param pkts_n
+ *   Number of packets to be stored.
+ */
+static inline void
+rxq_copy_mbuf_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t n)
+{
+	const uint16_t q_mask = (1 << rxq->elts_n) - 1;
+	struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask];
+	unsigned int pos;
+	uint16_t p = n & -2;
+
+	for (pos = 0; pos < p; pos += 2) {
+		__m128i mbp;
+
+		mbp = _mm_loadu_si128((__m128i *)&elts[pos]);
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp);
+	}
+	if (n & 1)
+		pkts[pos] = elts[pos];
+}
+
+/**
+ * Replenish buffers for RX in bulk.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ */
+static inline void
+rxq_replenish_bulk_mbuf(struct rxq *rxq)
+{
+	const uint16_t q_n = 1 << rxq->elts_n;
+	const uint16_t q_mask = q_n - 1;
+	const uint16_t elts_idx = rxq->rq_ci & q_mask;
+	struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
+	volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx];
+	uint16_t n = q_n - (rxq->rq_ci - rxq->rq_pi);
+	unsigned int i;
+
+	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH);
+	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
+	if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
+		rxq->stats.rx_nombuf += n;
+		return;
+	}
+	for (i = 0; i < n; ++i)
+		wq[i].addr = htonll(rte_pktmbuf_mtod(elts[i], uintptr_t));
+	rxq->rq_ci += n;
+	rte_wmb();
+	*rxq->rq_db = htonl(rxq->rq_ci);
+}
+
+/**
+ * Decompress a compressed CQE and fill in mbufs in RX SW ring with data
+ * extracted from the title CQE.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cq
+ *   Pointer to CQ array having a compressed CQE at first.
+ * @param elts
+ *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
+ *   the title CQE to be copied to the rest of mbufs.
+ */
+static inline void
+rxq_cq_decompress_v(struct rxq *rxq,
+		    volatile struct mlx5_cqe *cq,
+		    struct rte_mbuf **elts)
+{
+	volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + 1);
+	struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */
+	unsigned int pos;
+	unsigned int i;
+	unsigned int inv = 0;
+	/* Mask to shuffle from extracted mini CQE to mbuf. */
+	const __m128i shuf_mask1 =
+		_mm_set_epi8(0,  1,  2,  3, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			     6,  7,         /* data_len, bswap16 */
+			    -1, -1,  6,  7, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	const __m128i shuf_mask2 =
+		_mm_set_epi8(8,  9, 10, 11, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			    14, 15,         /* data_len, bswap16 */
+			    -1, -1, 14, 15, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	/* Restore the compressed count. Must be 16 bits. */
+	const uint16_t mcqe_n = t_pkt->data_len +
+				(rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i rearm =
+		_mm_loadu_si128((__m128i *)&t_pkt->rearm_data);
+	const __m128i rxdf =
+		_mm_loadu_si128((__m128i *)&t_pkt->rx_descriptor_fields1);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0, 0);
+	const uint32_t flow_tag = t_pkt->hash.fdir.hi;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     14, 15,  6,  7,
+			     10, 11,  2,  3);
+#endif
+
+	/* Compile time sanity check for this function. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+	/*
+	 * A. load mCQEs into a 128bit register.
+	 * B. store rearm data to mbuf.
+	 * C. combine data from mCQEs with rx_descriptor_fields1.
+	 * D. store rx_descriptor_fields1.
+	 * E. store flow tag (rte_flow mark).
+	 */
+	for (pos = 0; pos < mcqe_n; ) {
+		__m128i mcqe1, mcqe2;
+		__m128i rxdf1, rxdf2;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt, invalid_mask;
+#endif
+
+		if (!(pos & 0x7) && pos + 8 < mcqe_n)
+			rte_prefetch0((void *)(cq + pos + 8));
+		/* A.1 load mCQEs into a 128bit register. */
+		mcqe1 = _mm_loadu_si128((__m128i *)&mcq[pos % 8]);
+		mcqe2 = _mm_loadu_si128((__m128i *)&mcq[pos % 8 + 2]);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 1]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe1, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe1, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 1]->rx_descriptor_fields1,
+				 rxdf2);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos + 2]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 3]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe2, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe2, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 2]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 3]->rx_descriptor_fields1,
+				 rxdf2);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		invalid_mask = ones << ((mcqe_n - pos) * sizeof(uint16_t) * 8);
+		mcqe1 = _mm_srli_si128(mcqe1, 4);
+		byte_cnt = _mm_blend_epi16(mcqe1, mcqe2, 0xcc);
+		byte_cnt = _mm_shuffle_epi8(byte_cnt, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		if (rxq->mark) {
+			/* E.1 store flow tag (rte_flow mark). */
+			elts[pos]->hash.fdir.hi = flow_tag;
+			elts[pos + 1]->hash.fdir.hi = flow_tag;
+			elts[pos + 2]->hash.fdir.hi = flow_tag;
+			elts[pos + 3]->hash.fdir.hi = flow_tag;
+		}
+		pos += MLX5_VPMD_DESCS_PER_LOOP;
+		/* Move to next CQE and invalidate consumed CQEs. */
+		if (!(pos & 0x7) && pos < mcqe_n) {
+			mcq = (void *)(cq + pos);
+			for (i = 0; i < 8; ++i)
+				cq[inv++].op_own = MLX5_CQE_INVALIDATE;
+		}
+	}
+	/* Invalidate the rest of CQEs. */
+	for (; inv < mcqe_n; ++inv)
+		cq[inv].op_own = MLX5_CQE_INVALIDATE;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += mcqe_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	rxq->cq_ci += mcqe_n;
+}
+
+/*
+ * The index to the array should have:
+ * bit[1:0] = l3_hdr_type, bit[2] = tunneled, bit[3] = outer_l3_type
+ */
+static uint32_t mlx5_ptype_table[] = {
+	RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b0001 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b0010 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b0101 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b0110 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b1001 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b1010 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b1101 */
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b1110 */
+	RTE_PTYPE_ALL_MASK			     /* b1111 */
+};
+
+/**
+ * Calculate packet type and offload flag for mbuf and store it.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cqes[4]
+ *   Array of four 16bytes CQEs extracted from the original CQE.
+ * @param op_err
+ *   Opcode vector having responder error status. Each field is 4B.
+ * @param pkts
+ *   Pointer to array of packets to be filled.
+ */
+static inline void
+rxq_cq_to_ptype_oflags_v(struct rxq *rxq, __m128i cqes[4], __m128i op_err,
+			 struct rte_mbuf **pkts)
+{
+	__m128i pinfo0;
+	__m128i pinfo1;
+	__m128i pinfo, ptype;
+	__m128i ol_flags = _mm_set1_epi32(
+			rxq->rss_hash * PKT_RX_RSS_HASH);
+	__m128i cv_flags;
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ptype_mask =
+		_mm_set_epi32(0xd06, 0xd06, 0xd06, 0xd06);
+	const __m128i ptype_ol_mask =
+		_mm_set_epi32(0x106, 0x106, 0x106, 0x106);
+	const __m128i pinfo_mask =
+		_mm_set_epi32(0x3, 0x3, 0x3, 0x3);
+	const __m128i cv_flag_sel =
+		_mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0,
+			     (uint8_t)((PKT_RX_IP_CKSUM_GOOD |
+					     PKT_RX_L4_CKSUM_GOOD) >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1),
+			     (uint8_t)(PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED),
+			     0);
+	const __m128i cv_mask =
+		_mm_set_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED);
+	const __m128i mbuf_init =
+		_mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer);
+	__m128i rearm0, rearm1, rearm2, rearm3;
+
+	/* Extract pkt_info field. */
+	pinfo0 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+	pinfo = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	/* Extract hdr_type_etc field. */
+	pinfo0 = _mm_unpackhi_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpackhi_epi32(cqes[2], cqes[3]);
+	ptype = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	if (rxq->mark) {
+		const __m128i pinfo_ft_mask =
+			_mm_set_epi32(0xffffff00, 0xffffff00,
+				      0xffffff00, 0xffffff00);
+		const __m128i fdir_flags = _mm_set1_epi32(PKT_RX_FDIR);
+		const __m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID);
+		__m128i flow_tag, invalid_mask;
+
+		flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask);
+		/* Check if flow tag is non-zero then set PKT_RX_FDIR. */
+		invalid_mask = _mm_cmpeq_epi32(flow_tag, zero);
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(invalid_mask,
+							 fdir_flags));
+		/* Mask out invalid entries. */
+		flow_tag = _mm_andnot_si128(invalid_mask, flow_tag);
+		/* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(
+						_mm_cmpeq_epi32(flow_tag,
+								pinfo_ft_mask),
+						fdir_id_flags));
+	}
+	/*
+	 * Merge the two fields to generate the following:
+	 * bit[1]  = l3_ok,    bit[2]     = l4_ok
+	 * bit[8]  = cv,       bit[11:10] = l3_hdr_type
+	 * bit[12] = tunneled, bit[13]    = outer_l3_type
+	 */
+	ptype = _mm_and_si128(ptype, ptype_mask);
+	pinfo = _mm_and_si128(pinfo, pinfo_mask);
+	pinfo = _mm_slli_epi32(pinfo, 12);
+	ptype = _mm_or_si128(ptype, pinfo);
+	ptype = _mm_srli_epi32(ptype, 10);
+	ptype = _mm_packs_epi32(ptype, zero);
+	/* Errored packets will have RTE_PTYPE_ALL_MASK. */
+	op_err = _mm_srli_epi16(op_err, 12);
+	ptype = _mm_or_si128(ptype, op_err);
+	pkts[0]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 0)];
+	pkts[1]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 2)];
+	pkts[2]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 4)];
+	pkts[3]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 6)];
+	/* Fill flags for checksum and VLAN. */
+	pinfo = _mm_and_si128(pinfo, ptype_ol_mask);
+	pinfo = _mm_shuffle_epi8(cv_flag_sel, pinfo);
+	/* Locate checksum flags at byte[2:1] and merge with VLAN flags. */
+	cv_flags = _mm_slli_epi32(pinfo, 9);
+	cv_flags = _mm_or_si128(pinfo, cv_flags);
+	/* Move back flags to start from byte[0]. */
+	cv_flags = _mm_srli_epi32(cv_flags, 8);
+	/* Mask out garbage bits. */
+	cv_flags = _mm_and_si128(cv_flags, cv_mask);
+	/* Merge to ol_flags. */
+	ol_flags = _mm_or_si128(ol_flags, cv_flags);
+	/* Merge mbuf_init and ol_flags. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+			 offsetof(struct rte_mbuf, rearm_data) + 8);
+	rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 8), 0x30);
+	rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 4), 0x30);
+	rearm2 = _mm_blend_epi16(mbuf_init, ol_flags, 0x30);
+	rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(ol_flags, 4), 0x30);
+	/* Write 8B rearm_data and 8B ol_flags. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+			 RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+	_mm_store_si128((__m128i *)&pkts[0]->rearm_data, rearm0);
+	_mm_store_si128((__m128i *)&pkts[1]->rearm_data, rearm1);
+	_mm_store_si128((__m128i *)&pkts[2]->rearm_data, rearm2);
+	_mm_store_si128((__m128i *)&pkts[3]->rearm_data, rearm3);
+}
+
+/**
+ * Skip error packets.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+static uint16_t
+rxq_handle_pending_error(struct rxq *rxq, struct rte_mbuf **pkts,
+			 uint16_t pkts_n)
+{
+	uint16_t n = 0;
+	unsigned int i;
+
+	for (i = 0; i < pkts_n; ++i) {
+		struct rte_mbuf *pkt = pkts[i];
+
+		if (pkt->packet_type == RTE_PTYPE_ALL_MASK)
+			rte_pktmbuf_free_seg(pkt);
+		else
+			pkts[n++] = pkt;
+	}
+	rxq->stats.idropped += (pkts_n - n);
+	rxq->pending_err = 0;
+	return n;
+}
+
+/**
+ * Receive burst of packets. An errored CQE also consumes a mbuf, but the
+ * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed
+ * before returning to application.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets received including errors (<= pkts_n).
+ */
+static inline uint16_t
+rxq_burst_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	const uint16_t q_n = 1 << rxq->cqe_n;
+	const uint16_t q_mask = q_n - 1;
+	volatile struct mlx5_cqe *cq;
+	struct rte_mbuf **elts;
+	unsigned int pos;
+	uint64_t n;
+	uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
+	uint16_t nocmp_n = 0;
+	uint16_t rcvd_pkt = 0;
+	unsigned int cq_idx = rxq->cq_ci & q_mask;
+	unsigned int elts_idx;
+	unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1));
+	const __m128i owner_check =
+		_mm_set_epi64x(0x0100000001000000LL, 0x0100000001000000LL);
+	const __m128i opcode_check =
+		_mm_set_epi64x(0xf0000000f0000000LL, 0xf0000000f0000000LL);
+	const __m128i format_check =
+		_mm_set_epi64x(0x0c0000000c000000LL, 0x0c0000000c000000LL);
+	const __m128i resp_err_check =
+		_mm_set_epi64x(0xe0000000e0000000LL, 0xe0000000e0000000LL);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     12, 13,  8,  9,
+			      4,  5,  0,  1);
+#endif
+	/* Mask to shuffle from extracted CQE to mbuf. */
+	const __m128i shuf_mask =
+		_mm_set_epi8(-1,  3,  2,  1, /* fdir.hi */
+			     12, 13, 14, 15, /* rss, bswap32 */
+			     10, 11,         /* vlan_tci, bswap16 */
+			      4,  5,         /* data_len, bswap16 */
+			     -1, -1,         /* zero out 2nd half of pkt_len */
+			      4,  5          /* pkt_len, bswap16 */);
+	/* Mask to blend from the last Qword to the first DQword. */
+	const __m128i blend_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			      0,  0,  0,  0,
+			      0,  0,  0, -1);
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0);
+
+	/* Compile time sanity check for this function. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, pkt_info) != 0);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rx_hash_res) !=
+			 offsetof(struct mlx5_cqe, pkt_info) + 12);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd1) +
+			  sizeof(((struct mlx5_cqe *)0)->rsvd1) !=
+			 offsetof(struct mlx5_cqe, hdr_type_etc));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, vlan_info) !=
+			 offsetof(struct mlx5_cqe, hdr_type_etc) + 2);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd2) +
+			  sizeof(((struct mlx5_cqe *)0)->rsvd2) !=
+			 offsetof(struct mlx5_cqe, byte_cnt));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, sop_drop_qpn) !=
+			 RTE_ALIGN(offsetof(struct mlx5_cqe, sop_drop_qpn), 8));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, op_own) !=
+			 offsetof(struct mlx5_cqe, sop_drop_qpn) + 7);
+	assert(rxq->sges_n == 0);
+	assert(rxq->cqe_n == rxq->elts_n);
+	cq = &(*rxq->cqes)[cq_idx];
+	rte_prefetch0(cq);
+	rte_prefetch0(cq + 1);
+	rte_prefetch0(cq + 2);
+	rte_prefetch0(cq + 3);
+	pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
+	/*
+	 * Order of indexes:
+	 *   rq_ci >= cq_ci >= rq_pi
+	 * Definition of indexes:
+	 *   rq_ci - cq_ci := # of buffers owned by HW (posted).
+	 *   cq_ci - rq_pi := # of buffers not returned to app (decompressed).
+	 *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
+	 */
+	if ((uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)) >=
+			MLX5_VPMD_RXQ_RPLNSH_THRESH)
+		rxq_replenish_bulk_mbuf(rxq);
+	/* See if there're unreturned mbufs from compressed CQE. */
+	rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
+	if (rcvd_pkt > 0) {
+		rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
+		rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt);
+		rxq->rq_pi += rcvd_pkt;
+		pkts += rcvd_pkt;
+	}
+	elts_idx = rxq->rq_pi & q_mask;
+	elts = &(*rxq->elts)[elts_idx];
+	/* Not to overflow pkts array. */
+	pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
+	if (!pkts_n)
+		return rcvd_pkt;
+	/* At this point, there shouldn't be any remained packets. */
+	assert(rxq->rq_pi == rxq->cq_ci);
+	/*
+	 * A. load first Qword (8bytes) in one loop.
+	 * B. copy 4 mbuf pointers from elts ring to returing pkts.
+	 * C. load remained CQE data and extract necessary fields.
+	 *    Final 16bytes cqes[] extracted from original 64bytes CQE has the
+	 *    following structure:
+	 *        struct {
+	 *          uint8_t  pkt_info;
+	 *          uint8_t  flow_tag[3];
+	 *          uint16_t byte_cnt;
+	 *          uint8_t  rsvd4;
+	 *          uint8_t  op_own;
+	 *          uint16_t hdr_type_etc;
+	 *          uint16_t vlan_info;
+	 *          uint32_t rx_has_res;
+	 *        } c;
+	 * D. fill in mbuf.
+	 * E. get valid CQEs.
+	 * F. find compressed CQE.
+	 */
+	for (pos = 0;
+	     pos < pkts_n;
+	     pos += MLX5_VPMD_DESCS_PER_LOOP) {
+		__m128i cqes[MLX5_VPMD_DESCS_PER_LOOP];
+		__m128i cqe_tmp1, cqe_tmp2;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
+		__m128i op_own, op_own_tmp1, op_own_tmp2;
+		__m128i opcode, owner_mask, invalid_mask;
+		__m128i comp_mask;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt;
+#endif
+		__m128i mbp1, mbp2;
+		__m128i p = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0);
+		unsigned int p1, p2, p3;
+
+		/* Prefetch next 4 CQEs. */
+		if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) {
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]);
+		}
+		/* A.0 Do not cross the end of CQ. */
+		p = _mm_andnot_si128(ones << ((pkts_n - pos) *
+					      sizeof(uint16_t) * 8), p);
+		/* A.1 load cqes. */
+		p3 = _mm_extract_epi16(p, 3);
+		cqes[3] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p3].sop_drop_qpn);
+		rte_compiler_barrier();
+		p2 = _mm_extract_epi16(p, 2);
+		cqes[2] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p2].sop_drop_qpn);
+		rte_compiler_barrier();
+		/* B.1 load mbuf pointers. */
+		mbp1 = _mm_loadu_si128((__m128i *)&elts[pos]);
+		mbp2 = _mm_loadu_si128((__m128i *)&elts[pos + 2]);
+		/* A.1 load a block having op_own. */
+		p1 = _mm_extract_epi16(p, 1);
+		cqes[1] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p1].sop_drop_qpn);
+		rte_compiler_barrier();
+		cqes[0] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos].sop_drop_qpn);
+		/* B.2 copy mbuf pointers. */
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp1);
+		_mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2);
+		rte_compiler_barrier();
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]);
+		cqes[3] = _mm_blendv_epi8(cqes[3], cqe_tmp2, blend_mask);
+		cqes[2] = _mm_blendv_epi8(cqes[2], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p3].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos + p2].rsvd1[3]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x30);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p3].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos + p2].rsvd2[10]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x04);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb3 = _mm_shuffle_epi8(cqes[3], shuf_mask);
+		pkt_mb2 = _mm_shuffle_epi8(cqes[2], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb3 = _mm_sub_epi16(pkt_mb3, crc_adj);
+		pkt_mb2 = _mm_sub_epi16(pkt_mb2, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb3 = _mm_add_epi32(pkt_mb3, flow_mark_adj);
+		pkt_mb2 = _mm_add_epi32(pkt_mb2, flow_mark_adj);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 3]->pkt_len, pkt_mb3);
+		_mm_storeu_si128((void *)&pkts[pos + 2]->pkt_len, pkt_mb2);
+		/* E.1 extract op_own field. */
+		op_own_tmp2 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p1]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos]);
+		cqes[1] = _mm_blendv_epi8(cqes[1], cqe_tmp2, blend_mask);
+		cqes[0] = _mm_blendv_epi8(cqes[0], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p1].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos].rsvd1[3]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x30);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p1].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos].rsvd2[10]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x04);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb1 = _mm_shuffle_epi8(cqes[1], shuf_mask);
+		pkt_mb0 = _mm_shuffle_epi8(cqes[0], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb1 = _mm_sub_epi16(pkt_mb1, crc_adj);
+		pkt_mb0 = _mm_sub_epi16(pkt_mb0, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb1 = _mm_add_epi32(pkt_mb1, flow_mark_adj);
+		pkt_mb0 = _mm_add_epi32(pkt_mb0, flow_mark_adj);
+		/* E.1 extract op_own byte. */
+		op_own_tmp1 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+		op_own = _mm_unpackhi_epi64(op_own_tmp1, op_own_tmp2);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 1]->pkt_len, pkt_mb1);
+		_mm_storeu_si128((void *)&pkts[pos]->pkt_len, pkt_mb0);
+		/* E.2 flip owner bit to mark CQEs from last round. */
+		owner_mask = _mm_and_si128(op_own, owner_check);
+		if (ownership)
+			owner_mask = _mm_xor_si128(owner_mask, owner_check);
+		owner_mask = _mm_cmpeq_epi32(owner_mask, owner_check);
+		owner_mask = _mm_packs_epi32(owner_mask, zero);
+		/* E.3 get mask for invalidated CQEs. */
+		opcode = _mm_and_si128(op_own, opcode_check);
+		invalid_mask = _mm_cmpeq_epi32(opcode_check, opcode);
+		invalid_mask = _mm_packs_epi32(invalid_mask, zero);
+		/* E.4 mask out beyond boundary. */
+		invalid_mask =
+			_mm_or_si128(invalid_mask,
+				     ones << ((pkts_n - pos) *
+					      sizeof(uint16_t) * 8));
+		/* E.5 merge invalid_mask with invalid owner. */
+		invalid_mask = _mm_or_si128(invalid_mask, owner_mask);
+		/* F.1 find compressed CQE format. */
+		comp_mask = _mm_and_si128(op_own, format_check);
+		comp_mask = _mm_cmpeq_epi32(comp_mask, format_check);
+		comp_mask = _mm_packs_epi32(comp_mask, zero);
+		/* F.2 mask out invalid entries. */
+		comp_mask = _mm_andnot_si128(invalid_mask, comp_mask);
+		comp_idx = _mm_cvtsi128_si64(comp_mask);
+		/* F.3 get the first compressed CQE. */
+		comp_idx = comp_idx ?
+				__builtin_ctzll(comp_idx) /
+					(sizeof(uint16_t) * 8) :
+				MLX5_VPMD_DESCS_PER_LOOP;
+		/* E.6 mask out entries after the compressed CQE. */
+		invalid_mask =
+			_mm_or_si128(invalid_mask,
+				     ones << (comp_idx *
+					      sizeof(uint16_t) * 8));
+		/* E.7 count non-compressed valid CQEs. */
+		n = _mm_cvtsi128_si64(invalid_mask);
+		n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) :
+			MLX5_VPMD_DESCS_PER_LOOP;
+		nocmp_n += n;
+		/* D.2 get the final invalid mask. */
+		invalid_mask =
+			_mm_or_si128(invalid_mask,
+				     ones << (n * sizeof(uint16_t) * 8));
+		/* D.3 check error in opcode. */
+		opcode = _mm_cmpeq_epi32(resp_err_check, opcode);
+		opcode = _mm_packs_epi32(opcode, zero);
+		opcode = _mm_andnot_si128(invalid_mask, opcode);
+		/* D.4 mark if any error is set */
+		rxq->pending_err |= !!_mm_cvtsi128_si64(opcode);
+		/* D.5 fill in mbuf - rearm_data and packet_type. */
+		rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Add up received bytes count. */
+		byte_cnt = _mm_shuffle_epi8(op_own, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		/*
+		 * Break the loop unless more valid CQE is expected, or if
+		 * there's a compressed CQE.
+		 */
+		if (n != MLX5_VPMD_DESCS_PER_LOOP)
+			break;
+	}
+	/* If no new CQE seen, return without updating cq_db. */
+	if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP))
+		return rcvd_pkt;
+	/* Update the consumer indexes for non-compressed CQEs. */
+	assert(nocmp_n <= pkts_n);
+	rxq->cq_ci += nocmp_n;
+	rxq->rq_pi += nocmp_n;
+	rcvd_pkt += nocmp_n;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += nocmp_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	/* Decompress the last CQE if compressed. */
+	if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) {
+		assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
+		rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]);
+		/* Return more packets if needed. */
+		if (nocmp_n < pkts_n) {
+			uint16_t n = rxq->cq_ci - rxq->rq_pi;
+
+			n = RTE_MIN(n, pkts_n - nocmp_n);
+			rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n);
+			rxq->rq_pi += n;
+			rcvd_pkt += n;
+		}
+	}
+	rte_wmb();
+	*rxq->cq_db = htonl(rxq->cq_ci);
+	return rcvd_pkt;
+}
+
+/**
+ * DPDK callback for vectorized RX.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct rxq *rxq = dpdk_rxq;
+	uint16_t nb_rx;
+
+	nb_rx = rxq_burst_v(rxq, pkts, pkts_n);
+	if (unlikely(rxq->pending_err))
+		nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
+	return nb_rx;
+}
+
+/**
+ * Check a TX queue can support vectorized TX.
+ *
+ * @param txq
+ *   Pointer to TX queue.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+txq_check_vec_tx_support(struct txq *txq)
+{
+	/* Currently unused, but for possible future use. */
+	(void)txq;
+	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check Tx queue flags are set for raw vectorized Tx.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_raw_vec_tx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->txqs_n; ++i) {
+		struct txq *txq = (*priv->txqs)[i];
+
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) ||
+		    !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
+			break;
+	}
+	if (i != priv->txqs_n)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized TX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_vec_tx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->txqs_n; ++i) {
+		struct txq *txq = (*priv->txqs)[i];
+
+		if (txq_check_vec_tx_support(txq) < 0)
+			break;
+	}
+	if (i != priv->txqs_n)
+		return -ENOTSUP;
+	if (priv->txqs_n > MLX5_VPMD_MIN_TXQS ||
+	    priv->mps != MLX5_MPW_ENHANCED ||
+	    priv->tso)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a RX queue can support vectorized RX.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+rxq_check_vec_rx_support(struct rxq *rxq)
+{
+	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1) ||
+	    rxq->sges_n != 0)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized RX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_vec_rx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+
+		if (rxq_check_vec_rx_support(rxq) < 0)
+			break;
+	}
+	if (i != priv->rxqs_n)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Prepare for vectorized RX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+void
+priv_prep_vec_rx_function(struct priv *priv)
+{
+	uint16_t i;
+
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+		struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
+		const uint16_t desc = 1 << rxq->elts_n;
+		int j;
+
+		assert(rxq->elts_n == rxq->cqe_n);
+		/* Initialize default rearm_data for vPMD. */
+		mbuf_init->data_off = RTE_PKTMBUF_HEADROOM;
+		rte_mbuf_refcnt_set(mbuf_init, 1);
+		mbuf_init->nb_segs = 1;
+		mbuf_init->port = rxq->port_id;
+		/*
+		 * prevent compiler reordering:
+		 * rearm_data covers previous fields.
+		 */
+		rte_compiler_barrier();
+		rxq->mbuf_initializer =
+			*(uint64_t *)&mbuf_init->rearm_data;
+		/* Padding with a fake mbuf for vectorized Rx. */
+		for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j)
+			(*rxq->elts)[desc + j] = &rxq->fake_mbuf;
+		/* Mark that it need to be cleaned up for rxq_alloc_elts(). */
+		rxq->trim_elts = 1;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 03e23f58d..3ad6ac0c6 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -243,7 +243,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
 		ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set");
 		goto error;
 	}
-	(void)conf; /* Thresholds configuration (ignored). */
+	tmpl.txq.flags = conf->txq_flags;
 	assert(desc > MLX5_TX_COMP_THRESH);
 	tmpl.txq.elts_n = log2above(desc);
 	if (priv->mps == MLX5_MPW_ENHANCED)
-- 
2.11.0

Re: [dpdk-dev] [PATCH v2 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Thomas Monjalon]

30/06/2017 21:23, Yongseok Koh:
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -39,6 +39,11 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c
> +ifneq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),)
> +ifeq ($(CONFIG_RTE_ARCH_X86_64),y)
> +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx_vec_sse.c
> +endif
> +endif
[...]
> +#check if flag for SSE4.1 is already on, if not set it up manually
> +ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),)
> +CFLAGS_mlx5_rxtx_vec_sse.o += -msse4.1
> +endif

You can remove this check and SSE flag because SSE4.2 is now
the minimum requirement for DPDK on x86:
	http://dpdk.org/patch/25444

Re: [dpdk-dev] [PATCH v2 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Nélio Laranjeiro]

Yongseok, some comments in this huge and great work,

On Fri, Jun 30, 2017 at 12:23:33PM -0700, Yongseok Koh wrote:
> To make vectorized burst routines enabled, it is required to run on x86_64
> architecture which can support at least SSE4.1. If all the conditions are
> met, the vectorized burst functions are enabled automatically. The decision
> is made individually on RX and TX. There's no PMD option to make a
> selection.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
>  drivers/net/mlx5/Makefile            |   10 +
>  drivers/net/mlx5/mlx5_defs.h         |   18 +
>  drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
>  drivers/net/mlx5/mlx5_rxq.c          |   55 +-
>  drivers/net/mlx5/mlx5_rxtx.c         |  339 ++------
>  drivers/net/mlx5/mlx5_rxtx.h         |  283 ++++++-
>  drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/mlx5_txq.c          |    2 +-
>  8 files changed, 1909 insertions(+), 277 deletions(-)
>  create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c
> 
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index 51e258a15..2d0894fcd 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
>[...]
>  
> +#ifndef NDEBUG
> +
> +/**
> + * Verify or set magic value in CQE.
> + *
> + * @param cqe
> + *   Pointer to CQE.
> + *
> + * @return
> + *   0 the first time.
> + */
> +static inline int
> +check_cqe_seen(volatile struct mlx5_cqe *cqe)
>[...]
> +/**
> + * Check whether CQE is valid.
> + *
> + * @param cqe
> + *   Pointer to CQE.
> + * @param cqes_n
> + *   Size of completion queue.
> + * @param ci
> + *   Consumer index.
> + *
> + * @return
> + *   0 on success, 1 on failure.
> + */
> +static inline int
> +check_cqe(volatile struct mlx5_cqe *cqe,
> +	  unsigned int cqes_n, const uint16_t ci)
>[...]
> +
> +/**
> + * Return the address of the WQE.
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + * @param  wqe_ci
> + *   WQE consumer index.
> + *
> + * @return
> + *   WQE address.
> + */
> +static inline uintptr_t *
> +tx_mlx5_wqe(struct txq *txq, uint16_t ci)
>[...]
> +
> +/**
> + * Manage TX completions.
> + *
> + * When sending a burst, mlx5_tx_burst() posts several WRs.
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + */
> +static inline void
> +txq_complete(struct txq *txq)
>[...]
> +
> +/**
> + * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
> + * the cloned mbuf is allocated is returned instead.
> + *
> + * @param buf
> + *   Pointer to mbuf.
> + *
> + * @return
> + *   Memory pool where data is located for given mbuf.
> + */
> +static struct rte_mempool *
> +txq_mb2mp(struct rte_mbuf *buf)
>[...]
> +
> +/**
> + * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
> + * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
> + * remove an entry first.
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + * @param[in] mp
> + *   Memory Pool for which a Memory Region lkey must be returned.
> + *
> + * @return
> + *   mr->lkey on success, (uint32_t)-1 on failure.
> + */
> +static inline uint32_t
> +txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
>[...]

Most of the function moved above should be prefix as there some have the
same name in mlx4.  Static linkage will face some issues when both are
compiled.

> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.c b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
> new file mode 100644
> index 000000000..566a60635
> --- /dev/null
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
> @@ -0,0 +1,1451 @@
>[...]
> +
> +#include <assert.h>
> +#include <stdint.h>
> +#include <string.h>
> +#include <stdlib.h>
> +
> +/* Verbs header. */
> +/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
> +#ifdef PEDANTIC
> +#pragma GCC diagnostic ignored "-Wpedantic"
> +#endif
> +#include <infiniband/verbs.h>
> +#include <infiniband/mlx5_hw.h>
> +#include <infiniband/arch.h>
> +#ifdef PEDANTIC
> +#pragma GCC diagnostic error "-Wpedantic"
> +#endif
> +
> +/* DPDK headers don't like -pedantic. */
> +#ifdef PEDANTIC
> +#pragma GCC diagnostic ignored "-Wpedantic"
> +#endif
> +#include <rte_mbuf.h>
> +#include <rte_mempool.h>
> +#include <rte_prefetch.h>
> +#ifdef PEDANTIC
> +#pragma GCC diagnostic error "-Wpedantic"
> +#endif

Pragmas are not more necessary on DPDK headers.

> +
> +#include "mlx5.h"
> +#include "mlx5_utils.h"
> +#include "mlx5_rxtx.h"
> +#include "mlx5_autoconf.h"
> +#include "mlx5_defs.h"
> +#include "mlx5_prm.h"
> +
> +#include <smmintrin.h>

This include should be present with the system includes block to follow
a hierarchy:

 System
 Verbs libs
 DPDK libs
 PMD header.

> +#ifndef __INTEL_COMPILER
> +#pragma GCC diagnostic ignored "-Wcast-qual"
> +#endif
> +
> +/**
> + * Fill in DSEGs in Tx descriptors (WQE).

Unless for people whom work in Mellanox PMD, DSEG and WQE does not mean
anything.  You should avoid such acronyms in comments.

>[...]
> +/**
> + * Send multi-segmented packets until it encounters a single segment packet in
> + * the pkts list.
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + * @param pkts
> + *   Pointer to array of packets to be sent.
> + * @param pkts_n
> + *   Number of packets to be sent.
> + *
> + * @return
> + *   Number of packets successfully transmitted (<= pkts_n).
> + */
> +static uint16_t
> +txq_scatter_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n)
> +{
> +	uint16_t elts_head = txq->elts_head;
> +	const uint16_t elts_n = 1 << txq->elts_n;
> +	const uint16_t elts_m = elts_n - 1;
> +	const uint16_t wq_n = 1 << txq->wqe_n;
> +	const uint16_t wq_mask = wq_n - 1;
> +	const unsigned int nb_dword_per_wqebb =
> +		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
> +	const unsigned int nb_dword_in_hdr =
> +		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
> +	unsigned int n;
> +	volatile struct mlx5_wqe *wqe = NULL;
> +
> +	assert(elts_n > pkts_n);
> +	txq_complete(txq);
> +	/* A CQE slot must always be available. */
> +	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));

This assert should be moved to the txq_complete(), or it should not be
an assert.

>[...]
> +/**
> + * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
> + * it returns to make it processed by txq_scatter_v(). All the packets in
> + * the pkts list should be single segment packets having same offload flags.
> + * This must be checked by txq_check_multiseg() and txq_calc_offload().
> + *
> + * @param txq
> + *   Pointer to TX queue structure.
> + * @param pkts
> + *   Pointer to array of packets to be sent.
> + * @param pkts_n
> + *   Number of packets to be sent.

A small comment here telling pkts_n <= MLX5_VPMD_TX_MAX_BURST would
remove the assert few lines later.  The caller must respect the
contract.

> + * @param cs_flags
> + *   Checksum offload flags to be written in the descriptor.
> + *
> + * @return
> + *   Number of packets successfully transmitted (<= pkts_n).
> + */
> +static inline uint16_t
> +txq_burst_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
> +	    uint8_t cs_flags)
> +{
> +	struct rte_mbuf **elts;
> +	uint16_t elts_head = txq->elts_head;
> +	const uint16_t elts_n = 1 << txq->elts_n;
> +	const uint16_t elts_m = elts_n - 1;
> +	const unsigned int nb_dword_per_wqebb =
> +		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
> +	const unsigned int nb_dword_in_hdr =
> +		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
> +	unsigned int n = 0;
> +	unsigned int pos;
> +	uint16_t max_elts;
> +	uint16_t max_wqe;
> +	uint32_t comp_req = 0;
> +	const uint16_t wq_n = 1 << txq->wqe_n;
> +	const uint16_t wq_mask = wq_n - 1;
> +	uint16_t wq_idx = txq->wqe_ci & wq_mask;
> +	volatile struct mlx5_wqe64 *wq =
> +		&((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx];
> +	volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq;
> +	const __m128i shuf_mask_ctrl =
> +		_mm_set_epi8(15, 14, 13, 12,
> +			      8,  9, 10, 11, /* bswap32 */
> +			      4,  5,  6,  7, /* bswap32 */
> +			      0,  1,  2,  3  /* bswap32 */);
> +	__m128i *t_wqe, *dseg;
> +	__m128i ctrl;
> +
> +	/* Make sure all packets can fit into a single WQE. */
> +	assert(pkts_n <= MLX5_VPMD_TX_MAX_BURST);
> +	assert(elts_n > pkts_n);
> +	txq_complete(txq);
> +	max_elts = (elts_n - (elts_head - txq->elts_tail));
> +	/* A CQE slot must always be available. */
> +	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));

Same point here, this assert should be in txq_complete().

>[...] 
> +/**
> + * DPDK callback for vectorized TX with multi-seg packets and offload.
> + *
> + * @param dpdk_txq
> + *   Generic pointer to TX queue structure.
> + * @param[in] pkts
> + *   Packets to transmit.
> + * @param pkts_n
> + *   Number of packets in array.
> + *
> + * @return
> + *   Number of packets successfully transmitted (<= pkts_n).
> + */
> +uint16_t
> +mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
> +{
> +	struct txq *txq = (struct txq *)dpdk_txq;
> +	uint16_t nb_tx = 0;
> +
> +	while (pkts_n > nb_tx) {
> +		uint8_t cs_flags = 0;
> +		uint16_t n;
> +		uint16_t ret;
> +
> +		/* Transmit multi-seg packets in the head of pkts list. */
> +		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) &&
> +		    NB_SEGS(pkts[nb_tx]) > 1)
> +			nb_tx += txq_scatter_v(
> +					txq, &pkts[nb_tx], pkts_n - nb_tx);

The indentation here does not respect the PMD code style.

> +		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
> +		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS))
> +			n = txq_check_multiseg(&pkts[nb_tx], n);
> +		if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
> +			n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags);
> +		ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
> +		nb_tx += ret;
> +		if (!ret)
> +			break;
> +	}
> +	return nb_tx;
> +}
>[...]
> +/**
> + * Replenish buffers for RX in bulk.
> + *
> + * @param rxq
> + *   Pointer to RX queue structure.
> + */
> +static inline void
> +rxq_replenish_bulk_mbuf(struct rxq *rxq)
> +{
> +	const uint16_t q_n = 1 << rxq->elts_n;
> +	const uint16_t q_mask = q_n - 1;
> +	const uint16_t elts_idx = rxq->rq_ci & q_mask;
> +	struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
> +	volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx];
> +	uint16_t n = q_n - (rxq->rq_ci - rxq->rq_pi);

This computation is already performed in the parent, couldn't it be also
passed as parameter to this function?

> +	unsigned int i;
> +
> +	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH);
> +	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP);
> +	/* Not to cross queue end. */
> +	n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
> +	if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
> +		rxq->stats.rx_nombuf += n;
> +		return;
> +	}
> +	for (i = 0; i < n; ++i)
> +		wq[i].addr = htonll(rte_pktmbuf_mtod(elts[i], uintptr_t));
> +	rxq->rq_ci += n;
> +	rte_wmb();
> +	*rxq->rq_db = htonl(rxq->rq_ci);
> +}
> +
>[...]
> +/*
> + * The index to the array should have:
> + * bit[1:0] = l3_hdr_type, bit[2] = tunneled, bit[3] = outer_l3_type
> + */
> +static uint32_t mlx5_ptype_table[] = {
> +	RTE_PTYPE_UNKNOWN,
> +	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b0001 */
> +	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b0010 */
> +	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
> +	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
> +		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b0101 */
> +	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
> +		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b0110 */
> +	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
> +	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b1001 */
> +	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b1010 */
> +	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
> +	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
> +		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b1101 */
> +	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
> +		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b1110 */
> +	RTE_PTYPE_ALL_MASK			     /* b1111 */
> +};

This table could be shared with the regular Rx function.

> +/**
> + * Calculate packet type and offload flag for mbuf and store it.
> + *
> + * @param rxq
> + *   Pointer to RX queue structure.
> + * @param cqes[4]
> + *   Array of four 16bytes CQEs extracted from the original CQE.
> + * @param op_err
> + *   Opcode vector having responder error status. Each field is 4B.
> + * @param pkts
> + *   Pointer to array of packets to be filled.
> + */
> +static inline void
> +rxq_cq_to_ptype_oflags_v(struct rxq *rxq, __m128i cqes[4], __m128i op_err,
> +			 struct rte_mbuf **pkts)
> +{
> +	__m128i pinfo0;
> +	__m128i pinfo1;
> +	__m128i pinfo, ptype;
> +	__m128i ol_flags = _mm_set1_epi32(
> +			rxq->rss_hash * PKT_RX_RSS_HASH);

This two can be on the same line.

> +	__m128i cv_flags;
> +	const __m128i zero = _mm_setzero_si128();
> +	const __m128i ptype_mask =
> +		_mm_set_epi32(0xd06, 0xd06, 0xd06, 0xd06);
> +	const __m128i ptype_ol_mask =
> +		_mm_set_epi32(0x106, 0x106, 0x106, 0x106);
> +	const __m128i pinfo_mask =
> +		_mm_set_epi32(0x3, 0x3, 0x3, 0x3);
> +	const __m128i cv_flag_sel =
> +		_mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0,
> +			     (uint8_t)((PKT_RX_IP_CKSUM_GOOD |
> +					     PKT_RX_L4_CKSUM_GOOD) >> 1),

The alignment is wrong.

> +			     0,
> +			     (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1),
> +			     0,
> +			     (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1),
> +			     (uint8_t)(PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED),
> +			     0);
> +	const __m128i cv_mask =
> +		_mm_set_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
> +				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
> +			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
> +				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
> +			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
> +				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
> +			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
> +				PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED);

Same here.

> +	const __m128i mbuf_init =
> +		_mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer);
> +	__m128i rearm0, rearm1, rearm2, rearm3;
>[...]
> +/**
> + * Check a TX queue can support vectorized TX.
> + *
> + * @param txq
> + *   Pointer to TX queue.
> + *
> + * @return
> + *   1 if supported, negative errno value if not.
> + */
> +int __attribute__((cold))
> +txq_check_vec_tx_support(struct txq *txq)
> +{
> +	/* Currently unused, but for possible future use. */

This comment is not useful as the PMD code style prefix reflects the
first parameter of the function, it is expected from the function name
to have txq as first parameter even if not use (for now).

> +	(void)txq;
> +	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
> +		return -ENOTSUP;
> +	return 1;
> +}
> +
>[...]

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v2 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Yongseok Koh]

On Tue, Jul 04, 2017 at 10:58:52AM +0200, Nélio Laranjeiro wrote:
> Yongseok, some comments in this huge and great work,
> 
> On Fri, Jun 30, 2017 at 12:23:33PM -0700, Yongseok Koh wrote:
> > To make vectorized burst routines enabled, it is required to run on x86_64
> > architecture which can support at least SSE4.1. If all the conditions are
> > met, the vectorized burst functions are enabled automatically. The decision
> > is made individually on RX and TX. There's no PMD option to make a
> > selection.
> > 
> > Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> > ---
> >  drivers/net/mlx5/Makefile            |   10 +
> >  drivers/net/mlx5/mlx5_defs.h         |   18 +
> >  drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
> >  drivers/net/mlx5/mlx5_rxq.c          |   55 +-
> >  drivers/net/mlx5/mlx5_rxtx.c         |  339 ++------
> >  drivers/net/mlx5/mlx5_rxtx.h         |  283 ++++++-
> >  drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
> >  drivers/net/mlx5/mlx5_txq.c          |    2 +-
> >  8 files changed, 1909 insertions(+), 277 deletions(-)
> >  create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c
> > 
> > diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> > index 51e258a15..2d0894fcd 100644
> > --- a/drivers/net/mlx5/mlx5_rxtx.h
> > +++ b/drivers/net/mlx5/mlx5_rxtx.h
[...]
> > +	txq_complete(txq);
> > +	/* A CQE slot must always be available. */
> > +	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
> 
> This assert should be moved to the txq_complete(), or it should not be
> an assert.
txq_complete() is a common function, so this can't force to spare at least one
slot in a completion queue. This assert is to force to allocate enough CQE slots
by accurate calculation as completion is suppressed by MLX5_TX_COMP_THRESH. If
the CQ size is well defined (e.g. size of Tx ring / MLX5_TX_COMP_THRESH), it
doesn't need to check deficiency of CQ slot but checking slots in Tx ring
(max_elts) is sufficient. If you are okay with this, please let me know, then
I'll send out v3.

And I agree on all other comments you made. I'll make changes accordingly in v3.

Thanks for your quick review!
Yongseok

Re: [dpdk-dev] [PATCH v2 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Nélio Laranjeiro]

On Tue, Jul 04, 2017 at 05:38:44PM -0700, Yongseok Koh wrote:
> On Tue, Jul 04, 2017 at 10:58:52AM +0200, Nélio Laranjeiro wrote:
> > Yongseok, some comments in this huge and great work,
> > 
> > On Fri, Jun 30, 2017 at 12:23:33PM -0700, Yongseok Koh wrote:
> > > To make vectorized burst routines enabled, it is required to run on x86_64
> > > architecture which can support at least SSE4.1. If all the conditions are
> > > met, the vectorized burst functions are enabled automatically. The decision
> > > is made individually on RX and TX. There's no PMD option to make a
> > > selection.
> > > 
> > > Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> > > ---
> > >  drivers/net/mlx5/Makefile            |   10 +
> > >  drivers/net/mlx5/mlx5_defs.h         |   18 +
> > >  drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
> > >  drivers/net/mlx5/mlx5_rxq.c          |   55 +-
> > >  drivers/net/mlx5/mlx5_rxtx.c         |  339 ++------
> > >  drivers/net/mlx5/mlx5_rxtx.h         |  283 ++++++-
> > >  drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
> > >  drivers/net/mlx5/mlx5_txq.c          |    2 +-
> > >  8 files changed, 1909 insertions(+), 277 deletions(-)
> > >  create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c
> > > 
> > > diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> > > index 51e258a15..2d0894fcd 100644
> > > --- a/drivers/net/mlx5/mlx5_rxtx.h
> > > +++ b/drivers/net/mlx5/mlx5_rxtx.h
> [...]
> > > +	txq_complete(txq);
> > > +	/* A CQE slot must always be available. */
> > > +	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
> > 
> > This assert should be moved to the txq_complete(), or it should not be
> > an assert.
> txq_complete() is a common function, so this can't force to spare at least one
> slot in a completion queue. This assert is to force to allocate enough CQE slots
> by accurate calculation as completion is suppressed by MLX5_TX_COMP_THRESH. If
> the CQ size is well defined (e.g. size of Tx ring / MLX5_TX_COMP_THRESH), it
> doesn't need to check deficiency of CQ slot but checking slots in Tx ring
> (max_elts) is sufficient. If you are okay with this, please let me know, then
> I'll send out v3.

Just using your comment...
 /* A CQE slot must always be available. */

This is always true in any Tx function where this assumption should be
true to avoid testing it before posting a completion request and thus
avoid cycles waste and this independently of the computation for the CQ
ring size.

As it is an assert it is only present to help the developer in error
code he developed (or for a user to point an issue in the code), this
can be any-where in the Tx data path.  It becomes useful for any Tx
function using this txq_complete().  

If this assert is only related to your code, it may means it should be
an if which avoids to post a completion request when no slots are
available.

> And I agree on all other comments you made. I'll make changes accordingly in v3.

Ok,

> 
> Thanks for your quick review!
> Yongseok

Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v2 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Yongseok Koh]

On Wed, Jul 05, 2017 at 10:21:26AM +0200, Nélio Laranjeiro wrote:
> On Tue, Jul 04, 2017 at 05:38:44PM -0700, Yongseok Koh wrote:
> > On Tue, Jul 04, 2017 at 10:58:52AM +0200, Nélio Laranjeiro wrote:
> > > Yongseok, some comments in this huge and great work,
> > > 
> > > On Fri, Jun 30, 2017 at 12:23:33PM -0700, Yongseok Koh wrote:
> > > > To make vectorized burst routines enabled, it is required to run on x86_64
> > > > architecture which can support at least SSE4.1. If all the conditions are
> > > > met, the vectorized burst functions are enabled automatically. The decision
> > > > is made individually on RX and TX. There's no PMD option to make a
> > > > selection.
> > > > 
> > > > Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> > > > ---
> > > >  drivers/net/mlx5/Makefile            |   10 +
> > > >  drivers/net/mlx5/mlx5_defs.h         |   18 +
> > > >  drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
> > > >  drivers/net/mlx5/mlx5_rxq.c          |   55 +-
> > > >  drivers/net/mlx5/mlx5_rxtx.c         |  339 ++------
> > > >  drivers/net/mlx5/mlx5_rxtx.h         |  283 ++++++-
> > > >  drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
> > > >  drivers/net/mlx5/mlx5_txq.c          |    2 +-
> > > >  8 files changed, 1909 insertions(+), 277 deletions(-)
> > > >  create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c
> > > > 
> > > > diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> > > > index 51e258a15..2d0894fcd 100644
> > > > --- a/drivers/net/mlx5/mlx5_rxtx.h
> > > > +++ b/drivers/net/mlx5/mlx5_rxtx.h
> > [...]
> > > > +	txq_complete(txq);
> > > > +	/* A CQE slot must always be available. */
> > > > +	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
> > > 
> > > This assert should be moved to the txq_complete(), or it should not be
> > > an assert.
> > txq_complete() is a common function, so this can't force to spare at least one
> > slot in a completion queue. This assert is to force to allocate enough CQE slots
> > by accurate calculation as completion is suppressed by MLX5_TX_COMP_THRESH. If
> > the CQ size is well defined (e.g. size of Tx ring / MLX5_TX_COMP_THRESH), it
> > doesn't need to check deficiency of CQ slot but checking slots in Tx ring
> > (max_elts) is sufficient. If you are okay with this, please let me know, then
> > I'll send out v3.
> 
> Just using your comment...
>  /* A CQE slot must always be available. */
> 
> This is always true in any Tx function where this assumption should be
> true to avoid testing it before posting a completion request and thus
> avoid cycles waste and this independently of the computation for the CQ
> ring size.
> 
> As it is an assert it is only present to help the developer in error
> code he developed (or for a user to point an issue in the code), this
> can be any-where in the Tx data path.  It becomes useful for any Tx
> function using this txq_complete().  
> 
> If this assert is only related to your code, it may means it should be
> an if which avoids to post a completion request when no slots are
> available.
I think I was wrong about the assert. If the Tx ring is full (max_elts == 0),
then there won't be any empty slot left in CQ. I'll remove the two asserts.

Thanks
Yongseok

Re: [dpdk-dev] [PATCH v2 0/5] net/mlx5: add vectorized Rx/Tx burst for x86

[Bruce Richardson]

On Fri, Jun 30, 2017 at 12:23:28PM -0700, Yongseok Koh wrote:
> This is to introduce more efficient Rx/Tx burst functions using SIMD
> instructions. Currently it is only supported by 64bit x86 having SSE4.1.
> 
> From functional perspective, Rx burst function is equivalent to the
> existing mlx5_rx_burst() except for scatter support, which will be added
> soon. Tx burst function supports multi-segment packets and offload flags
> unless it is disabled by txq_flags. However, disabling those features will
> bring a little higher performance.
> 
> v2:
> * Streamline redundant conditional clauses in txq_complete().
> * Remove the mempool pointer in txq->mp2mr structure.
> * Fix indentation and spacing.
> 
> Yongseok Koh (5):
>   net/mlx5: change indexing for Tx SW ring
>   net/mlx5: free buffers in bulk on Tx completion
>   net/mlx5: use buffer address for LKEY search
>   net/mlx5: select Rx/Tx callbacks when starting device
>   net/mlx5: add vectorized Rx/Tx burst for SSE4.1
> 
>  drivers/net/mlx5/Makefile            |   10 +
>  drivers/net/mlx5/mlx5_defs.h         |   18 +
>  drivers/net/mlx5/mlx5_ethdev.c       |   28 +-
>  drivers/net/mlx5/mlx5_mr.c           |   17 +-
>  drivers/net/mlx5/mlx5_rxq.c          |   57 +-
>  drivers/net/mlx5/mlx5_rxtx.c         |  425 +++-------
>  drivers/net/mlx5/mlx5_rxtx.h         |  291 ++++++-
>  drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1451 ++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/mlx5_trigger.c      |    3 +
>  drivers/net/mlx5/mlx5_txq.c          |   23 +-
>  10 files changed, 1980 insertions(+), 343 deletions(-)
>  create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c
>
FYI: I've upstreamed a patchset to bump the minimum specs for DPDK on
x86 to include SSE4.2, so that would lead to some simplification of your
code, I expect. See this set: http://dpdk.org/dev/patchwork/patch/25444/

Regards,
/Bruce

[dpdk-dev] [PATCH v3 0/5] net/mlx5: add vectorized Rx/Tx burst for x86

[Yongseok Koh]

This is to introduce more efficient Rx/Tx burst functions using SIMD
instructions. Currently it is only supported by 64bit x86 having SSE4.1.

>From functional perspective, Rx burst function is equivalent to the
existing mlx5_rx_burst() except for scatter support, which will be added
soon. Tx burst function supports multi-segment packets and offload flags
unless it is disabled by txq_flags. However, disabling those features will
bring a little higher performance.

v3:
* Remove requirement of SSE4.1 as DPDK now mandates SSE4.2 support at least.
* Bug fix in "net/mlx5: select Rx/Tx callbacks when starting device"
  - Need to re-select Rx burst func when chaning MTU size.
* Resolved an optimization issue of gcc-6 in rxq_burst_v()
  - Bit shift (<<) for 128b vector type is compiled differently. 'psllq' is
    needed instead of 'sal'.
* Minor changes to address what is mentioned by review.
  - Remove 'pragma' for PEDANTIC
  - Make mlx5_ptype_table global.
  - Change name of some inline funcs which also exist in mlx4 by the same name.
  - Fix comments and indentation/spacing.

v2:
* Streamline redundant conditional clauses in txq_complete().
* Remove the mempool pointer in txq->mp2mr structure.
* Fix indentation and spacing.

Yongseok Koh (5):
  net/mlx5: change indexing for Tx SW ring
  net/mlx5: free buffers in bulk on Tx completion
  net/mlx5: use buffer address for LKEY search
  net/mlx5: select Rx/Tx callbacks when starting device
  net/mlx5: add vectorized Rx/Tx burst for SSE4.1

 drivers/net/mlx5/Makefile            |    3 +
 drivers/net/mlx5/mlx5_defs.h         |   18 +
 drivers/net/mlx5/mlx5_ethdev.c       |   47 +-
 drivers/net/mlx5/mlx5_mr.c           |   17 +-
 drivers/net/mlx5/mlx5_rxq.c          |   57 +-
 drivers/net/mlx5/mlx5_rxtx.c         |  459 ++++-------
 drivers/net/mlx5/mlx5_rxtx.h         |  290 ++++++-
 drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1378 ++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_trigger.c      |    3 +
 drivers/net/mlx5/mlx5_txq.c          |   23 +-
 10 files changed, 1927 insertions(+), 368 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c

-- 
2.11.0

[dpdk-dev] [PATCH v3 1/5] net/mlx5: change indexing for Tx SW ring

[Yongseok Koh]

For Tx SW ring (txq->elts[]), indexes are kept and used in
txq->elts_head/tail. Because of this, one entry must always be left unused
and it also makes code complex. Changed to store counters instead of
indexes in order to make the code simpler and to reduce a few calculations.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 108 +++++++++++++++++--------------------------
 drivers/net/mlx5/mlx5_rxtx.h |   4 +-
 drivers/net/mlx5/mlx5_txq.c  |  16 +++----
 3 files changed, 52 insertions(+), 76 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 70314b393..43db06ad8 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -255,7 +255,8 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
 static inline void
 txq_complete(struct txq *txq)
 {
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	const unsigned int cqe_n = 1 << txq->cqe_n;
 	const unsigned int cqe_cnt = cqe_n - 1;
 	uint16_t elts_free = txq->elts_tail;
@@ -292,24 +293,23 @@ txq_complete(struct txq *txq)
 	ctrl = (volatile struct mlx5_wqe_ctrl *)
 		tx_mlx5_wqe(txq, txq->wqe_pi);
 	elts_tail = ctrl->ctrl3;
-	assert(elts_tail < (1 << txq->wqe_n));
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
 	/* Free buffers. */
 	while (elts_free != elts_tail) {
-		struct rte_mbuf *elt = (*txq->elts)[elts_free];
-		unsigned int elts_free_next =
-			(elts_free + 1) & (elts_n - 1);
-		struct rte_mbuf *elt_next = (*txq->elts)[elts_free_next];
+		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
+		struct rte_mbuf *elt_next =
+			(*txq->elts)[(elts_free + 1) & elts_m];
 
 #ifndef NDEBUG
 		/* Poisoning. */
-		memset(&(*txq->elts)[elts_free],
+		memset(&(*txq->elts)[elts_free & elts_m],
 		       0x66,
-		       sizeof((*txq->elts)[elts_free]));
+		       sizeof((*txq->elts)[elts_free & elts_m]));
 #endif
 		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
 		/* Only one segment needs to be freed. */
 		rte_pktmbuf_free_seg(elt);
-		elts_free = elts_free_next;
+		++elts_free;
 	}
 	txq->cq_ci = cq_ci;
 	txq->elts_tail = elts_tail;
@@ -409,12 +409,10 @@ int
 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
 {
 	struct txq *txq = tx_queue;
-	const unsigned int elts_n = 1 << txq->elts_n;
-	const unsigned int elts_cnt = elts_n - 1;
-	unsigned int used;
+	uint16_t used;
 
 	txq_complete(txq);
-	used = (txq->elts_head - txq->elts_tail) & elts_cnt;
+	used = txq->elts_head - txq->elts_tail;
 	if (offset < used)
 		return RTE_ETH_TX_DESC_FULL;
 	return RTE_ETH_TX_DESC_DONE;
@@ -488,11 +486,12 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
 	unsigned int k = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	unsigned int max_inline = txq->max_inline;
 	const unsigned int inline_en = !!max_inline && txq->inline_en;
 	uint16_t max_wqe;
@@ -509,9 +508,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(*pkts);
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
@@ -540,9 +537,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--segs_n;
 		if (unlikely(--max_wqe == 0))
 			break;
@@ -561,7 +558,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		if (length < (MLX5_WQE_DWORD_SIZE + 2))
 			break;
 		/* Update element. */
-		(*txq->elts)[elts_head] = buf;
+		(*txq->elts)[elts_head & elts_m] = buf;
 		/* Prefetch next buffer data. */
 		if (pkts_n - i > 1)
 			rte_prefetch0(
@@ -801,8 +798,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr,
 			naddr >> 32,
 		};
-		elts_head = (elts_head + 1) & (elts_n - 1);
-		(*txq->elts)[elts_head] = buf;
+		(*txq->elts)[++elts_head & elts_m] = buf;
 		++sg;
 		/* Advance counter only if all segs are successfully posted. */
 		if (sg < segs_n)
@@ -810,7 +806,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		else
 			j += sg;
 next_pkt:
-		elts_head = (elts_head + 1) & (elts_n - 1);
+		++elts_head;
 		++pkts;
 		++i;
 		/* Initialize known and common part of the WQE structure. */
@@ -853,7 +849,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Take a shortcut if nothing must be sent. */
 	if (unlikely((i + k) == 0))
 		return 0;
-	txq->elts_head = (txq->elts_head + i + j) & (elts_n - 1);
+	txq->elts_head += (i + j);
 	/* Check whether completion threshold has been reached. */
 	comp = txq->elts_comp + i + j + k;
 	if (comp >= MLX5_TX_COMP_THRESH) {
@@ -960,10 +956,11 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int comp;
 	struct mlx5_mpw mpw = {
@@ -977,15 +974,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
 		uint32_t cs_flags = 0;
@@ -995,12 +989,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--pkts_n;
 		/* Should we enable HW CKSUM offload */
 		if (buf->ol_flags &
@@ -1036,9 +1030,8 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			volatile struct mlx5_wqe_data_seg *dseg;
 			uintptr_t addr;
 
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
 			assert(buf);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			dseg = mpw.data.dseg[mpw.pkts_n];
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
@@ -1046,7 +1039,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 				.addr = htonll(addr),
 			};
-			elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 			length += DATA_LEN(buf);
 #endif
@@ -1057,7 +1049,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		assert(length == mpw.len);
 		if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
 			mlx5_mpw_close(txq, &mpw);
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
@@ -1175,10 +1166,11 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int comp;
 	unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE;
@@ -1205,12 +1197,9 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uintptr_t addr;
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
@@ -1221,12 +1210,12 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--pkts_n;
 		/*
 		 * Compute max_wqe in case less WQE were consumed in previous
@@ -1287,10 +1276,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 			do {
 				volatile struct mlx5_wqe_data_seg *dseg;
 
-				elts_head_next =
-					(elts_head + 1) & (elts_n - 1);
 				assert(buf);
-				(*txq->elts)[elts_head] = buf;
+				(*txq->elts)[elts_head++ & elts_m] = buf;
 				dseg = mpw.data.dseg[mpw.pkts_n];
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
@@ -1298,7 +1285,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 					.addr = htonll(addr),
 				};
-				elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 				length += DATA_LEN(buf);
 #endif
@@ -1315,9 +1301,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 			assert(mpw.state == MLX5_MPW_INL_STATE_OPENED);
 			assert(length <= inline_room);
 			assert(length == DATA_LEN(buf));
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			/* Maximum number of bytes before wrapping. */
 			max = ((((uintptr_t)(txq->wqes)) +
 				(1 << txq->wqe_n) *
@@ -1354,7 +1339,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				inline_room -= length;
 			}
 		}
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
@@ -1476,10 +1460,11 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max_elts;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE;
 	unsigned int mpw_room = 0;
@@ -1494,8 +1479,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Start processing. */
 	txq_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	if (max_elts > elts_n)
-		max_elts -= elts_n;
 	/* A CQE slot must always be available. */
 	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
@@ -1503,7 +1486,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		return 0;
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uintptr_t addr;
 		uint64_t naddr;
 		unsigned int n;
@@ -1517,7 +1499,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max_elts - j < segs_n + 1)
+		if (max_elts - j < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
@@ -1601,10 +1583,8 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			do {
 				volatile struct mlx5_wqe_data_seg *dseg;
 
-				elts_head_next =
-					(elts_head + 1) & (elts_n - 1);
 				assert(buf);
-				(*txq->elts)[elts_head] = buf;
+				(*txq->elts)[elts_head++ & elts_m] = buf;
 				dseg = mpw.data.dseg[mpw.pkts_n];
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
@@ -1612,7 +1592,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 					.addr = htonll(addr),
 				};
-				elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 				length += DATA_LEN(buf);
 #endif
@@ -1663,7 +1642,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			/* No need to get completion as the entire packet is
 			 * copied to WQ. Free the buf right away.
 			 */
-			elts_head_next = elts_head;
 			rte_pktmbuf_free_seg(buf);
 			mpw_room -= (inl_pad + sizeof(inl_hdr) + length);
 			/* Add pad in the next packet if any. */
@@ -1686,8 +1664,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				dseg = (volatile void *)
 					((uintptr_t)mpw.data.raw +
 					 inl_pad);
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			for (n = 0; n * RTE_CACHE_LINE_SIZE < length; n++)
 				rte_prefetch2((void *)(addr +
@@ -1706,7 +1683,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			mpw_room -= (inl_pad + sizeof(*dseg));
 			inl_pad = 0;
 		}
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 4195665f8..a21995030 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -242,8 +242,8 @@ struct hash_rxq {
 /* TX queue descriptor. */
 __extension__
 struct txq {
-	uint16_t elts_head; /* Current index in (*elts)[]. */
-	uint16_t elts_tail; /* First element awaiting completion. */
+	uint16_t elts_head; /* Current counter in (*elts)[]. */
+	uint16_t elts_tail; /* Counter of first element awaiting completion. */
 	uint16_t elts_comp; /* Counter since last completion request. */
 	uint16_t mpw_comp; /* WQ index since last completion request. */
 	uint16_t cq_ci; /* Consumer index for completion queue. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index bf72468d5..f0729a2a8 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -103,9 +103,10 @@ txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n)
 static void
 txq_free_elts(struct txq_ctrl *txq_ctrl)
 {
-	unsigned int elts_n = 1 << txq_ctrl->txq.elts_n;
-	unsigned int elts_head = txq_ctrl->txq.elts_head;
-	unsigned int elts_tail = txq_ctrl->txq.elts_tail;
+	const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	uint16_t elts_head = txq_ctrl->txq.elts_head;
+	uint16_t elts_tail = txq_ctrl->txq.elts_tail;
 	struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts;
 
 	DEBUG("%p: freeing WRs", (void *)txq_ctrl);
@@ -114,18 +115,17 @@ txq_free_elts(struct txq_ctrl *txq_ctrl)
 	txq_ctrl->txq.elts_comp = 0;
 
 	while (elts_tail != elts_head) {
-		struct rte_mbuf *elt = (*elts)[elts_tail];
+		struct rte_mbuf *elt = (*elts)[elts_tail & elts_m];
 
 		assert(elt != NULL);
 		rte_pktmbuf_free_seg(elt);
 #ifndef NDEBUG
 		/* Poisoning. */
-		memset(&(*elts)[elts_tail],
+		memset(&(*elts)[elts_tail & elts_m],
 		       0x77,
-		       sizeof((*elts)[elts_tail]));
+		       sizeof((*elts)[elts_tail & elts_m]));
 #endif
-		if (++elts_tail == elts_n)
-			elts_tail = 0;
+		++elts_tail;
 	}
 }
 
-- 
2.11.0

[dpdk-dev] [PATCH v3 2/5] net/mlx5: free buffers in bulk on Tx completion

[Yongseok Koh]

When processing Tx completion, it is more efficient to free buffers in bulk
using rte_mempool_put_bulk() if buffers are from a same mempool.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 43db06ad8..66593679f 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -264,6 +264,9 @@ txq_complete(struct txq *txq)
 	uint16_t cq_ci = txq->cq_ci;
 	volatile struct mlx5_cqe *cqe = NULL;
 	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	struct rte_mempool *pool = NULL;
+	unsigned int blk_n = 0;
 
 	do {
 		volatile struct mlx5_cqe *tmp;
@@ -296,21 +299,33 @@ txq_complete(struct txq *txq)
 	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
 	/* Free buffers. */
 	while (elts_free != elts_tail) {
-		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
-		struct rte_mbuf *elt_next =
-			(*txq->elts)[(elts_free + 1) & elts_m];
-
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (likely(m->pool == pool)) {
+				free[blk_n++] = m;
+			} else {
+				if (likely(pool != NULL))
+					rte_mempool_put_bulk(pool,
+							     (void *)free,
+							     blk_n);
+				free[0] = m;
+				pool = m->pool;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(pool, (void *)free, blk_n);
 #ifndef NDEBUG
-		/* Poisoning. */
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
 		memset(&(*txq->elts)[elts_free & elts_m],
 		       0x66,
 		       sizeof((*txq->elts)[elts_free & elts_m]));
-#endif
-		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
-		/* Only one segment needs to be freed. */
-		rte_pktmbuf_free_seg(elt);
 		++elts_free;
 	}
+#endif
 	txq->cq_ci = cq_ci;
 	txq->elts_tail = elts_tail;
 	/* Update the consumer index. */
-- 
2.11.0

[dpdk-dev] [PATCH v3 3/5] net/mlx5: use buffer address for LKEY search

[Yongseok Koh]

When searching LKEY, if search key is mempool pointer, the 2nd cacheline
has to be accessed and it even requires to check whether a buffer is
indirect per every search. Instead, using address for search key can reduce
cycles taken. And caching the last hit entry is beneficial as well.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_mr.c   | 17 ++++++++++++++---
 drivers/net/mlx5/mlx5_rxtx.c | 39 +++++++++++++++++++++------------------
 drivers/net/mlx5/mlx5_rxtx.h |  4 +++-
 drivers/net/mlx5/mlx5_txq.c  |  3 +--
 4 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 0a3638460..287335179 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -207,7 +207,8 @@ txq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx)
 			 sizeof(txq_ctrl->txq.mp2mr[0])));
 	}
 	/* Store the new entry. */
-	txq_ctrl->txq.mp2mr[idx].mp = mp;
+	txq_ctrl->txq.mp2mr[idx].start = (uintptr_t)mr->addr;
+	txq_ctrl->txq.mp2mr[idx].end = (uintptr_t)mr->addr + mr->length;
 	txq_ctrl->txq.mp2mr[idx].mr = mr;
 	txq_ctrl->txq.mp2mr[idx].lkey = htonl(mr->lkey);
 	DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
@@ -265,18 +266,28 @@ txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
 	struct txq_mp2mr_mbuf_check_data data = {
 		.ret = 0,
 	};
+	uintptr_t start;
+	uintptr_t end;
 	unsigned int i;
 
 	/* Register mempool only if the first element looks like a mbuf. */
 	if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
 			data.ret == -1)
 		return;
+	if (mlx5_check_mempool(mp, &start, &end) != 0) {
+		ERROR("mempool %p: not virtually contiguous",
+		      (void *)mp);
+		return;
+	}
 	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
-		if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) {
+		struct ibv_mr *mr = txq_ctrl->txq.mp2mr[i].mr;
+
+		if (unlikely(mr == NULL)) {
 			/* Unknown MP, add a new MR for it. */
 			break;
 		}
-		if (txq_ctrl->txq.mp2mr[i].mp == mp)
+		if (start >= (uintptr_t)mr->addr &&
+		    end <= (uintptr_t)mr->addr + mr->length)
 			return;
 	}
 	txq_mp2mr_reg(&txq_ctrl->txq, mp, i);
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 66593679f..688ee9028 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -77,7 +77,7 @@ static __rte_always_inline void
 txq_complete(struct txq *txq);
 
 static __rte_always_inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp);
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb);
 
 static __rte_always_inline void
 mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe);
@@ -352,7 +352,7 @@ txq_mb2mp(struct rte_mbuf *buf)
 }
 
 /**
- * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[].
+ * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
  * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
  * remove an entry first.
  *
@@ -365,27 +365,30 @@ txq_mb2mp(struct rte_mbuf *buf)
  *   mr->lkey on success, (uint32_t)-1 on failure.
  */
 static inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
 {
-	unsigned int i;
-	uint32_t lkey = (uint32_t)-1;
+	uint16_t i = txq->mr_cache_idx;
+	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
 
+	assert(i < RTE_DIM(txq->mp2mr));
+	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
+		return txq->mp2mr[i].lkey;
 	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
-		if (unlikely(txq->mp2mr[i].mp == NULL)) {
+		if (unlikely(txq->mp2mr[i].mr == NULL)) {
 			/* Unknown MP, add a new MR for it. */
 			break;
 		}
-		if (txq->mp2mr[i].mp == mp) {
+		if (txq->mp2mr[i].start <= addr &&
+		    txq->mp2mr[i].end >= addr) {
 			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
 			assert(htonl(txq->mp2mr[i].mr->lkey) ==
 			       txq->mp2mr[i].lkey);
-			lkey = txq->mp2mr[i].lkey;
-			break;
+			txq->mr_cache_idx = i;
+			return txq->mp2mr[i].lkey;
 		}
 	}
-	if (unlikely(lkey == (uint32_t)-1))
-		lkey = txq_mp2mr_reg(txq, mp, i);
-	return lkey;
+	txq->mr_cache_idx = 0;
+	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
 }
 
 /**
@@ -770,7 +773,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t){
 				htonl(length),
-				txq_mp2mr(txq, txq_mb2mp(buf)),
+				txq_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
@@ -809,7 +812,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		naddr = htonll(rte_pktmbuf_mtod(buf, uintptr_t));
 		*dseg = (rte_v128u32_t){
 			htonl(length),
-			txq_mp2mr(txq, txq_mb2mp(buf)),
+			txq_mb2mr(txq, buf),
 			naddr,
 			naddr >> 32,
 		};
@@ -1051,7 +1054,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
 				.byte_count = htonl(DATA_LEN(buf)),
-				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+				.lkey = txq_mb2mr(txq, buf),
 				.addr = htonll(addr),
 			};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1297,7 +1300,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+					.lkey = txq_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1604,7 +1607,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+					.lkey = txq_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1687,7 +1690,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t) {
 				htonl(length),
-				txq_mp2mr(txq, txq_mb2mp(buf)),
+				txq_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index a21995030..d0f508e90 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -267,10 +267,12 @@ struct txq {
 	volatile uint32_t *cq_db; /* Completion queue doorbell. */
 	volatile void *bf_reg; /* Blueflame register. */
 	struct {
-		const struct rte_mempool *mp; /* Cached Memory Pool. */
+		uintptr_t start; /* Start address of MR */
+		uintptr_t end; /* End address of MR */
 		struct ibv_mr *mr; /* Memory Region (for mp). */
 		uint32_t lkey; /* htonl(mr->lkey) */
 	} mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MP to MR translation table. */
+	uint16_t mr_cache_idx; /* Index of last hit entry. */
 	struct rte_mbuf *(*elts)[]; /* TX elements. */
 	struct mlx5_txq_stats stats; /* TX queue counters. */
 } __rte_cache_aligned;
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index f0729a2a8..ac9dfc5f0 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -149,9 +149,8 @@ txq_cleanup(struct txq_ctrl *txq_ctrl)
 	if (txq_ctrl->cq != NULL)
 		claim_zero(ibv_destroy_cq(txq_ctrl->cq));
 	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
-		if (txq_ctrl->txq.mp2mr[i].mp == NULL)
+		if (txq_ctrl->txq.mp2mr[i].mr == NULL)
 			break;
-		assert(txq_ctrl->txq.mp2mr[i].mr != NULL);
 		claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr));
 	}
 	memset(txq_ctrl, 0, sizeof(*txq_ctrl));
-- 
2.11.0

[dpdk-dev] [PATCH v3 4/5] net/mlx5: select Rx/Tx callbacks when starting device

[Yongseok Koh]

The callbacks are global to a device but the seletion is made every queue
configuration, which is redundant.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/mlx5_ethdev.c  | 23 +++++++++++------------
 drivers/net/mlx5/mlx5_rxq.c     |  2 --
 drivers/net/mlx5/mlx5_trigger.c |  3 +++
 drivers/net/mlx5/mlx5_txq.c     |  2 --
 4 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 96bccd5fd..4a77eac50 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -923,8 +923,6 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	struct priv *priv = dev->data->dev_private;
 	int ret = 0;
 	unsigned int i;
-	uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) =
-		mlx5_rx_burst;
 	unsigned int max_frame_len;
 	int rehash;
 	int restart = priv->started;
@@ -944,7 +942,7 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	/* Temporarily replace RX handler with a fake one, assuming it has not
 	 * been copied elsewhere. */
 	dev->rx_pkt_burst = removed_rx_burst;
-	/* Make sure everyone has left mlx5_rx_burst() and uses
+	/* Make sure everyone has left dev->rx_pkt_burst() and uses
 	 * removed_rx_burst() instead. */
 	rte_wmb();
 	usleep(1000);
@@ -1018,17 +1016,13 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 		/* Double fault, disable RX. */
 		break;
 	}
-	/*
-	 * Use a safe RX burst function in case of error, otherwise mimic
-	 * mlx5_dev_start().
-	 */
+	/* Mimic mlx5_dev_start(). */
 	if (ret) {
 		ERROR("unable to reconfigure RX queues, RX disabled");
-		rx_func = removed_rx_burst;
 	} else if (restart &&
-		 !rehash &&
-		 !priv_create_hash_rxqs(priv) &&
-		 !priv_rehash_flows(priv)) {
+		   !rehash &&
+		   !priv_create_hash_rxqs(priv) &&
+		   !priv_rehash_flows(priv)) {
 		if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_NONE)
 			priv_fdir_enable(priv);
 		priv_dev_interrupt_handler_install(priv, dev);
@@ -1036,7 +1030,12 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	priv->mtu = mtu;
 	/* Burst functions can now be called again. */
 	rte_wmb();
-	dev->rx_pkt_burst = rx_func;
+	/*
+	 * Use a safe RX burst function in case of error, otherwise select RX
+	 * burst function again.
+	 */
+	if (!ret)
+		priv_select_rx_function(priv);
 out:
 	priv_unlock(priv);
 	assert(ret >= 0);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 22be999e8..98b20eb92 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1226,8 +1226,6 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		DEBUG("%p: adding RX queue %p to list",
 		      (void *)dev, (void *)rxq_ctrl);
 		(*priv->rxqs)[idx] = &rxq_ctrl->rxq;
-		/* Update receive callback. */
-		priv_select_rx_function(priv);
 	}
 	priv_unlock(priv);
 	return -ret;
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 40f23da93..595a9e064 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -72,6 +72,9 @@ mlx5_dev_start(struct rte_eth_dev *dev)
 		priv_unlock(priv);
 		return 0;
 	}
+	/* Update Rx/Tx callback. */
+	priv_select_tx_function(priv);
+	priv_select_rx_function(priv);
 	DEBUG("%p: allocating and configuring hash RX queues", (void *)dev);
 	err = priv_create_hash_rxqs(priv);
 	if (!err)
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index ac9dfc5f0..03e23f58d 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -496,8 +496,6 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		DEBUG("%p: adding TX queue %p to list",
 		      (void *)dev, (void *)txq_ctrl);
 		(*priv->txqs)[idx] = &txq_ctrl->txq;
-		/* Update send callback. */
-		priv_select_tx_function(priv);
 	}
 	priv_unlock(priv);
 	return -ret;
-- 
2.11.0

Re: [dpdk-dev] [PATCH v3 4/5] net/mlx5: select Rx/Tx callbacks when starting device

[Nélio Laranjeiro]

On Wed, Jul 05, 2017 at 11:12:27AM -0700, Yongseok Koh wrote:
> The callbacks are global to a device but the seletion is made every queue
> configuration, which is redundant.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

-- 
Nélio Laranjeiro
6WIND

[dpdk-dev] [PATCH v3 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Yongseok Koh]

To make vectorized burst routines enabled, it is required to run on x86_64
architecture which can support at least SSE4.1. If all the conditions are
met, the vectorized burst functions are enabled automatically. The decision
is made individually on RX and TX. There's no PMD option to make a
selection.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
 drivers/net/mlx5/Makefile            |    3 +
 drivers/net/mlx5/mlx5_defs.h         |   18 +
 drivers/net/mlx5/mlx5_ethdev.c       |   24 +-
 drivers/net/mlx5/mlx5_rxq.c          |   55 +-
 drivers/net/mlx5/mlx5_rxtx.c         |  387 +++-------
 drivers/net/mlx5/mlx5_rxtx.h         |  282 ++++++-
 drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1378 ++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_txq.c          |    2 +-
 8 files changed, 1852 insertions(+), 297 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 8b8f6eaac..8736de5d3 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -39,6 +39,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c
+ifeq ($(CONFIG_RTE_ARCH_X86_64),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx_vec_sse.c
+endif
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_trigger.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mac.c
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 201bb3362..a76bc6f65 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -89,4 +89,22 @@
 /* Maximum Packet headers size (L2+L3+L4) for TSO. */
 #define MLX5_MAX_TSO_HEADER 128
 
+/* Default minimum number of Tx queues for vectorized Tx. */
+#define MLX5_VPMD_MIN_TXQS 4
+
+/* Threshold of buffer replenishment for vectorized Rx. */
+#define MLX5_VPMD_RXQ_RPLNSH_THRESH   64U
+
+/* Maximum size of burst for vectorized Rx. */
+#define MLX5_VPMD_RX_MAX_BURST        MLX5_VPMD_RXQ_RPLNSH_THRESH
+
+/*
+ * Maximum size of burst for vectorized Tx. This is related to the maximum size
+ * of Enhaned MPW (eMPW) WQE as vectorized Tx is supported with eMPW.
+ */
+#define MLX5_VPMD_TX_MAX_BURST        32U
+
+/* Number of packets vectorized Rx can simultaneously process in a loop. */
+#define MLX5_VPMD_DESCS_PER_LOOP      4
+
 #endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 4a77eac50..39b17d4ec 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -723,7 +723,8 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 
 	};
 
-	if (dev->rx_pkt_burst == mlx5_rx_burst)
+	if (dev->rx_pkt_burst == mlx5_rx_burst ||
+	    dev->rx_pkt_burst == mlx5_rx_burst_vec)
 		return ptypes;
 	return NULL;
 }
@@ -1585,9 +1586,16 @@ priv_select_tx_function(struct priv *priv)
 	priv->dev->tx_pkt_burst = mlx5_tx_burst;
 	/* Select appropriate TX function. */
 	if (priv->mps == MLX5_MPW_ENHANCED) {
-		priv->dev->tx_pkt_burst =
-			mlx5_tx_burst_empw;
-		DEBUG("selected Enhanced MPW TX function");
+		if (priv_check_vec_tx_support(priv) > 0) {
+			if (priv_check_raw_vec_tx_support(priv) > 0)
+				priv->dev->tx_pkt_burst = mlx5_tx_burst_raw_vec;
+			else
+				priv->dev->tx_pkt_burst = mlx5_tx_burst_vec;
+			DEBUG("selected Enhanced MPW TX vectorized function");
+		} else {
+			priv->dev->tx_pkt_burst = mlx5_tx_burst_empw;
+			DEBUG("selected Enhanced MPW TX function");
+		}
 	} else if (priv->mps && priv->txq_inline) {
 		priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline;
 		DEBUG("selected MPW inline TX function");
@@ -1606,5 +1614,11 @@ priv_select_tx_function(struct priv *priv)
 void
 priv_select_rx_function(struct priv *priv)
 {
-	priv->dev->rx_pkt_burst = mlx5_rx_burst;
+	if (priv_check_vec_rx_support(priv) > 0) {
+		priv_prep_vec_rx_function(priv);
+		priv->dev->rx_pkt_burst = mlx5_rx_burst_vec;
+		DEBUG("selected RX vectorized function");
+	} else {
+		priv->dev->rx_pkt_burst = mlx5_rx_burst;
+	}
 }
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 98b20eb92..34ec95bc5 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -634,6 +634,41 @@ priv_rehash_flows(struct priv *priv)
 }
 
 /**
+ * Unlike regular Rx function, vPMD Rx doesn't replace mbufs immediately when
+ * receiving packets. Instead it replaces later in bulk. In rxq->elts[], entries
+ * from rq_pi to rq_ci are owned by device but the rest is already delivered to
+ * application. In order not to reuse those mbufs by rxq_alloc_elts(), this
+ * function must be called to replace used mbufs.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+static int
+rxq_trim_elts(struct rxq *rxq)
+{
+	const uint16_t q_n = (1 << rxq->elts_n);
+	const uint16_t q_mask = q_n - 1;
+	uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi);
+	uint16_t i;
+
+	if (!rxq->trim_elts)
+		return 0;
+	for (i = 0; i < used; ++i) {
+		struct rte_mbuf *buf;
+		buf = rte_pktmbuf_alloc(rxq->mp);
+		if (!buf)
+			return ENOMEM;
+		(*rxq->elts)[(rxq->rq_ci + i) & q_mask] = buf;
+	}
+	rxq->rq_pi = rxq->rq_ci;
+	rxq->trim_elts = 0;
+	return 0;
+}
+
+/**
  * Allocate RX queue elements.
  *
  * @param rxq_ctrl
@@ -800,6 +835,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl)
 		return err;
 	}
 	/* Snatch mbufs from original queue. */
+	claim_zero(rxq_trim_elts(&rxq_ctrl->rxq));
 	claim_zero(rxq_alloc_elts(rxq_ctrl, elts_n, rxq_ctrl->rxq.elts));
 	for (i = 0; i != elts_n; ++i) {
 		struct rte_mbuf *buf = (*rxq_ctrl->rxq.elts)[i];
@@ -860,6 +896,7 @@ rxq_setup(struct rxq_ctrl *tmpl)
 	tmpl->rxq.cqe_n = log2above(cq_info.cqe_cnt);
 	tmpl->rxq.cq_ci = 0;
 	tmpl->rxq.rq_ci = 0;
+	tmpl->rxq.rq_pi = 0;
 	tmpl->rxq.cq_db = cq_info.dbrec;
 	tmpl->rxq.wqes =
 		(volatile struct mlx5_wqe_data_seg (*)[])
@@ -993,7 +1030,12 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
 	if (priv->cqe_comp) {
 		attr.cq.comp_mask |= IBV_EXP_CQ_INIT_ATTR_FLAGS;
 		attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE;
-		cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
+		/*
+		 * For vectorized Rx, it must not be doubled in order to
+		 * make cq_ci and rq_ci aligned.
+		 */
+		if (rxq_check_vec_support(&tmpl.rxq) < 0)
+			cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
 	}
 	tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, tmpl.channel, 0,
 				    &attr.cq);
@@ -1103,7 +1145,9 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
 	if (rxq_ctrl->rxq.elts_n) {
 		assert(1 << rxq_ctrl->rxq.elts_n == desc);
 		assert(rxq_ctrl->rxq.elts != tmpl.rxq.elts);
-		ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts);
+		ret = rxq_trim_elts(&rxq_ctrl->rxq);
+		if (!ret)
+			ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts);
 	} else
 		ret = rxq_alloc_elts(&tmpl, desc, NULL);
 	if (ret) {
@@ -1165,6 +1209,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	struct priv *priv = dev->data->dev_private;
 	struct rxq *rxq = (*priv->rxqs)[idx];
 	struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
+	const uint16_t desc_pad = MLX5_VPMD_DESCS_PER_LOOP; /* For vPMD. */
 	int ret;
 
 	if (mlx5_is_secondary())
@@ -1198,7 +1243,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		if (rxq_ctrl->rxq.elts_n != log2above(desc)) {
 			rxq_ctrl = rte_realloc(rxq_ctrl,
 					       sizeof(*rxq_ctrl) +
-					       desc * sizeof(struct rte_mbuf *),
+					       (desc + desc_pad) *
+						sizeof(struct rte_mbuf *),
 					       RTE_CACHE_LINE_SIZE);
 			if (!rxq_ctrl) {
 				ERROR("%p: unable to reallocate queue index %u",
@@ -1209,7 +1255,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		}
 	} else {
 		rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl) +
-					     desc * sizeof(struct rte_mbuf *),
+					     (desc + desc_pad) *
+					      sizeof(struct rte_mbuf *),
 					     0, socket);
 		if (rxq_ctrl == NULL) {
 			ERROR("%p: unable to allocate queue index %u",
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 688ee9028..ab6df19eb 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -69,19 +69,6 @@
 #include "mlx5_defs.h"
 #include "mlx5_prm.h"
 
-static __rte_always_inline int
-check_cqe(volatile struct mlx5_cqe *cqe,
-	  unsigned int cqes_n, const uint16_t ci);
-
-static __rte_always_inline void
-txq_complete(struct txq *txq);
-
-static __rte_always_inline uint32_t
-txq_mb2mr(struct txq *txq, struct rte_mbuf *mb);
-
-static __rte_always_inline void
-mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe);
-
 static __rte_always_inline uint32_t
 rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe);
 
@@ -92,101 +79,29 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe,
 static __rte_always_inline uint32_t
 rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe);
 
-#ifndef NDEBUG
-
-/**
- * Verify or set magic value in CQE.
- *
- * @param cqe
- *   Pointer to CQE.
- *
- * @return
- *   0 the first time.
- */
-static inline int
-check_cqe_seen(volatile struct mlx5_cqe *cqe)
-{
-	static const uint8_t magic[] = "seen";
-	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
-	int ret = 1;
-	unsigned int i;
-
-	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
-		if (!ret || (*buf)[i] != magic[i]) {
-			ret = 0;
-			(*buf)[i] = magic[i];
-		}
-	return ret;
-}
-
-#endif /* NDEBUG */
-
-/**
- * Check whether CQE is valid.
- *
- * @param cqe
- *   Pointer to CQE.
- * @param cqes_n
- *   Size of completion queue.
- * @param ci
- *   Consumer index.
- *
- * @return
- *   0 on success, 1 on failure.
- */
-static inline int
-check_cqe(volatile struct mlx5_cqe *cqe,
-	  unsigned int cqes_n, const uint16_t ci)
-{
-	uint16_t idx = ci & cqes_n;
-	uint8_t op_own = cqe->op_own;
-	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
-	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
-
-	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
-		return 1; /* No CQE. */
-#ifndef NDEBUG
-	if ((op_code == MLX5_CQE_RESP_ERR) ||
-	    (op_code == MLX5_CQE_REQ_ERR)) {
-		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
-		uint8_t syndrome = err_cqe->syndrome;
-
-		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
-		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
-			return 0;
-		if (!check_cqe_seen(cqe))
-			ERROR("unexpected CQE error %u (0x%02x)"
-			      " syndrome 0x%02x",
-			      op_code, op_code, syndrome);
-		return 1;
-	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
-		   (op_code != MLX5_CQE_REQ)) {
-		if (!check_cqe_seen(cqe))
-			ERROR("unexpected CQE opcode %u (0x%02x)",
-			      op_code, op_code);
-		return 1;
-	}
-#endif /* NDEBUG */
-	return 0;
-}
-
-/**
- * Return the address of the WQE.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param  wqe_ci
- *   WQE consumer index.
- *
- * @return
- *   WQE address.
+/*
+ * The index to the array should have:
+ * bit[1:0] = l3_hdr_type, bit[2] = tunneled, bit[3] = outer_l3_type
  */
-static inline uintptr_t *
-tx_mlx5_wqe(struct txq *txq, uint16_t ci)
-{
-	ci &= ((1 << txq->wqe_n) - 1);
-	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
-}
+const uint32_t mlx5_ptype_table[] = {
+	RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b0001 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b0010 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b0101 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b0110 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b1001 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b1010 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b1101 */
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b1110 */
+	RTE_PTYPE_ALL_MASK			     /* b1111 */
+};
 
 /**
  * Return the size of tailroom of WQ.
@@ -245,174 +160,6 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
 }
 
 /**
- * Manage TX completions.
- *
- * When sending a burst, mlx5_tx_burst() posts several WRs.
- *
- * @param txq
- *   Pointer to TX queue structure.
- */
-static inline void
-txq_complete(struct txq *txq)
-{
-	const uint16_t elts_n = 1 << txq->elts_n;
-	const uint16_t elts_m = elts_n - 1;
-	const unsigned int cqe_n = 1 << txq->cqe_n;
-	const unsigned int cqe_cnt = cqe_n - 1;
-	uint16_t elts_free = txq->elts_tail;
-	uint16_t elts_tail;
-	uint16_t cq_ci = txq->cq_ci;
-	volatile struct mlx5_cqe *cqe = NULL;
-	volatile struct mlx5_wqe_ctrl *ctrl;
-	struct rte_mbuf *m, *free[elts_n];
-	struct rte_mempool *pool = NULL;
-	unsigned int blk_n = 0;
-
-	do {
-		volatile struct mlx5_cqe *tmp;
-
-		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
-		if (check_cqe(tmp, cqe_n, cq_ci))
-			break;
-		cqe = tmp;
-#ifndef NDEBUG
-		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
-			if (!check_cqe_seen(cqe))
-				ERROR("unexpected compressed CQE, TX stopped");
-			return;
-		}
-		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
-		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
-			if (!check_cqe_seen(cqe))
-				ERROR("unexpected error CQE, TX stopped");
-			return;
-		}
-#endif /* NDEBUG */
-		++cq_ci;
-	} while (1);
-	if (unlikely(cqe == NULL))
-		return;
-	txq->wqe_pi = ntohs(cqe->wqe_counter);
-	ctrl = (volatile struct mlx5_wqe_ctrl *)
-		tx_mlx5_wqe(txq, txq->wqe_pi);
-	elts_tail = ctrl->ctrl3;
-	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
-	/* Free buffers. */
-	while (elts_free != elts_tail) {
-		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
-		if (likely(m != NULL)) {
-			if (likely(m->pool == pool)) {
-				free[blk_n++] = m;
-			} else {
-				if (likely(pool != NULL))
-					rte_mempool_put_bulk(pool,
-							     (void *)free,
-							     blk_n);
-				free[0] = m;
-				pool = m->pool;
-				blk_n = 1;
-			}
-		}
-	}
-	if (blk_n)
-		rte_mempool_put_bulk(pool, (void *)free, blk_n);
-#ifndef NDEBUG
-	elts_free = txq->elts_tail;
-	/* Poisoning. */
-	while (elts_free != elts_tail) {
-		memset(&(*txq->elts)[elts_free & elts_m],
-		       0x66,
-		       sizeof((*txq->elts)[elts_free & elts_m]));
-		++elts_free;
-	}
-#endif
-	txq->cq_ci = cq_ci;
-	txq->elts_tail = elts_tail;
-	/* Update the consumer index. */
-	rte_wmb();
-	*txq->cq_db = htonl(cq_ci);
-}
-
-/**
- * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
- * the cloned mbuf is allocated is returned instead.
- *
- * @param buf
- *   Pointer to mbuf.
- *
- * @return
- *   Memory pool where data is located for given mbuf.
- */
-static struct rte_mempool *
-txq_mb2mp(struct rte_mbuf *buf)
-{
-	if (unlikely(RTE_MBUF_INDIRECT(buf)))
-		return rte_mbuf_from_indirect(buf)->pool;
-	return buf->pool;
-}
-
-/**
- * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
- * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
- * remove an entry first.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param[in] mp
- *   Memory Pool for which a Memory Region lkey must be returned.
- *
- * @return
- *   mr->lkey on success, (uint32_t)-1 on failure.
- */
-static inline uint32_t
-txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
-{
-	uint16_t i = txq->mr_cache_idx;
-	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
-
-	assert(i < RTE_DIM(txq->mp2mr));
-	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
-		return txq->mp2mr[i].lkey;
-	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
-		if (unlikely(txq->mp2mr[i].mr == NULL)) {
-			/* Unknown MP, add a new MR for it. */
-			break;
-		}
-		if (txq->mp2mr[i].start <= addr &&
-		    txq->mp2mr[i].end >= addr) {
-			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
-			assert(htonl(txq->mp2mr[i].mr->lkey) ==
-			       txq->mp2mr[i].lkey);
-			txq->mr_cache_idx = i;
-			return txq->mp2mr[i].lkey;
-		}
-	}
-	txq->mr_cache_idx = 0;
-	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
-}
-
-/**
- * Ring TX queue doorbell.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param wqe
- *   Pointer to the last WQE posted in the NIC.
- */
-static inline void
-mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
-{
-	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
-	volatile uint64_t *src = ((volatile uint64_t *)wqe);
-
-	rte_wmb();
-	*txq->qp_db = htonl(txq->wqe_ci);
-	/* Ensure ordering between DB record and BF copy. */
-	rte_wmb();
-	*dst = *src;
-}
-
-/**
  * DPDK callback to check the status of a tx descriptor.
  *
  * @param tx_queue
@@ -429,7 +176,7 @@ mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
 	struct txq *txq = tx_queue;
 	uint16_t used;
 
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	used = txq->elts_head - txq->elts_tail;
 	if (offset < used)
 		return RTE_ETH_TX_DESC_FULL;
@@ -525,7 +272,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Prefetch first packet cacheline. */
 	rte_prefetch0(*pkts);
 	/* Start processing. */
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
@@ -773,7 +520,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t){
 				htonl(length),
-				txq_mb2mr(txq, buf),
+				mlx5_tx_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
@@ -812,7 +559,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		naddr = htonll(rte_pktmbuf_mtod(buf, uintptr_t));
 		*dseg = (rte_v128u32_t){
 			htonl(length),
-			txq_mb2mr(txq, buf),
+			mlx5_tx_mb2mr(txq, buf),
 			naddr,
 			naddr >> 32,
 		};
@@ -991,7 +738,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
@@ -1054,7 +801,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
 				.byte_count = htonl(DATA_LEN(buf)),
-				.lkey = txq_mb2mr(txq, buf),
+				.lkey = mlx5_tx_mb2mr(txq, buf),
 				.addr = htonll(addr),
 			};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1214,7 +961,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	do {
 		struct rte_mbuf *buf = *(pkts++);
@@ -1300,7 +1047,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mb2mr(txq, buf),
+					.lkey = mlx5_tx_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1495,7 +1242,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	if (unlikely(!pkts_n))
 		return 0;
 	/* Start processing. */
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	/* A CQE slot must always be available. */
 	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
@@ -1607,7 +1354,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mb2mr(txq, buf),
+					.lkey = mlx5_tx_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1690,7 +1437,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t) {
 				htonl(length),
-				txq_mb2mr(txq, buf),
+				mlx5_tx_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
@@ -2138,3 +1885,71 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	(void)pkts_n;
 	return 0;
 }
+
+/*
+ * Vectorized Rx/Tx routines are not compiled in when required vector
+ * instructions are not supported on a target architecture. The following null
+ * stubs are needed for linkage when those are not included outside of this file
+ * (e.g.  mlx5_rxtx_vec_sse.c for x86).
+ */
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_txq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_txq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_rxq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+int __attribute__((weak))
+priv_check_raw_vec_tx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_vec_tx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+rxq_check_vec_support(struct rxq *rxq)
+{
+	(void)rxq;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_vec_rx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+void __attribute__((weak))
+priv_prep_vec_rx_function(struct priv *priv)
+{
+	(void)priv;
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index d0f508e90..534aaeb46 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -115,10 +115,13 @@ struct rxq {
 	unsigned int port_id:8;
 	unsigned int rss_hash:1; /* RSS hash result is enabled. */
 	unsigned int mark:1; /* Marked flow available on the queue. */
-	unsigned int :8; /* Remaining bits. */
+	unsigned int pending_err:1; /* CQE error needs to be handled. */
+	unsigned int trim_elts:1; /* Whether elts needs clean-up. */
+	unsigned int :6; /* Remaining bits. */
 	volatile uint32_t *rq_db;
 	volatile uint32_t *cq_db;
 	uint16_t rq_ci;
+	uint16_t rq_pi;
 	uint16_t cq_ci;
 	volatile struct mlx5_wqe_data_seg(*wqes)[];
 	volatile struct mlx5_cqe(*cqes)[];
@@ -126,6 +129,8 @@ struct rxq {
 	struct rte_mbuf *(*elts)[];
 	struct rte_mempool *mp;
 	struct mlx5_rxq_stats stats;
+	uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */
+	struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
 } __rte_cache_aligned;
 
 /* RX queue control descriptor. */
@@ -261,6 +266,7 @@ struct txq {
 	uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */
 	uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
 	uint32_t qp_num_8s; /* QP number shifted by 8. */
+	uint32_t flags; /* Flags for Tx Queue. */
 	volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
 	volatile void *wqes; /* Work queue (use volatile to write into). */
 	volatile uint32_t *qp_db; /* Work queue doorbell. */
@@ -328,6 +334,8 @@ uint16_t mlx5_tx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t);
 
 /* mlx5_rxtx.c */
 
+extern const uint32_t mlx5_ptype_table[];
+
 uint16_t mlx5_tx_burst(void *, struct rte_mbuf **, uint16_t);
 uint16_t mlx5_tx_burst_mpw(void *, struct rte_mbuf **, uint16_t);
 uint16_t mlx5_tx_burst_mpw_inline(void *, struct rte_mbuf **, uint16_t);
@@ -338,10 +346,282 @@ uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t);
 int mlx5_rx_descriptor_status(void *, uint16_t);
 int mlx5_tx_descriptor_status(void *, uint16_t);
 
+/* Vectorized version of mlx5_rxtx.c */
+int priv_check_raw_vec_tx_support(struct priv *);
+int priv_check_vec_tx_support(struct priv *);
+int rxq_check_vec_support(struct rxq *);
+int priv_check_vec_rx_support(struct priv *);
+void priv_prep_vec_rx_function(struct priv *);
+uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t);
+uint16_t mlx5_tx_burst_vec(void *, struct rte_mbuf **, uint16_t);
+uint16_t mlx5_rx_burst_vec(void *, struct rte_mbuf **, uint16_t);
+
 /* mlx5_mr.c */
 
 struct ibv_mr *mlx5_mp2mr(struct ibv_pd *, struct rte_mempool *);
 void txq_mp2mr_iter(struct rte_mempool *, void *);
 uint32_t txq_mp2mr_reg(struct txq *, struct rte_mempool *, unsigned int);
 
+#ifndef NDEBUG
+/**
+ * Verify or set magic value in CQE.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ *
+ * @return
+ *   0 the first time.
+ */
+static inline int
+check_cqe_seen(volatile struct mlx5_cqe *cqe)
+{
+	static const uint8_t magic[] = "seen";
+	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
+	int ret = 1;
+	unsigned int i;
+
+	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
+		if (!ret || (*buf)[i] != magic[i]) {
+			ret = 0;
+			(*buf)[i] = magic[i];
+		}
+	return ret;
+}
+#endif /* NDEBUG */
+
+/**
+ * Check whether CQE is valid.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ * @param cqes_n
+ *   Size of completion queue.
+ * @param ci
+ *   Consumer index.
+ *
+ * @return
+ *   0 on success, 1 on failure.
+ */
+static __rte_always_inline int
+check_cqe(volatile struct mlx5_cqe *cqe,
+	  unsigned int cqes_n, const uint16_t ci)
+{
+	uint16_t idx = ci & cqes_n;
+	uint8_t op_own = cqe->op_own;
+	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
+	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
+
+	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
+		return 1; /* No CQE. */
+#ifndef NDEBUG
+	if ((op_code == MLX5_CQE_RESP_ERR) ||
+	    (op_code == MLX5_CQE_REQ_ERR)) {
+		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
+		uint8_t syndrome = err_cqe->syndrome;
+
+		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
+		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
+			return 0;
+		if (!check_cqe_seen(cqe))
+			ERROR("unexpected CQE error %u (0x%02x)"
+			      " syndrome 0x%02x",
+			      op_code, op_code, syndrome);
+		return 1;
+	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
+		   (op_code != MLX5_CQE_REQ)) {
+		if (!check_cqe_seen(cqe))
+			ERROR("unexpected CQE opcode %u (0x%02x)",
+			      op_code, op_code);
+		return 1;
+	}
+#endif /* NDEBUG */
+	return 0;
+}
+
+/**
+ * Return the address of the WQE.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param  wqe_ci
+ *   WQE consumer index.
+ *
+ * @return
+ *   WQE address.
+ */
+static inline uintptr_t *
+tx_mlx5_wqe(struct txq *txq, uint16_t ci)
+{
+	ci &= ((1 << txq->wqe_n) - 1);
+	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
+}
+
+/**
+ * Manage TX completions.
+ *
+ * When sending a burst, mlx5_tx_burst() posts several WRs.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ */
+static __rte_always_inline void
+mlx5_tx_complete(struct txq *txq)
+{
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int cqe_n = 1 << txq->cqe_n;
+	const unsigned int cqe_cnt = cqe_n - 1;
+	uint16_t elts_free = txq->elts_tail;
+	uint16_t elts_tail;
+	uint16_t cq_ci = txq->cq_ci;
+	volatile struct mlx5_cqe *cqe = NULL;
+	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	struct rte_mempool *pool = NULL;
+	unsigned int blk_n = 0;
+
+	do {
+		volatile struct mlx5_cqe *tmp;
+
+		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
+		if (check_cqe(tmp, cqe_n, cq_ci))
+			break;
+		cqe = tmp;
+#ifndef NDEBUG
+		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
+			if (!check_cqe_seen(cqe))
+				ERROR("unexpected compressed CQE, TX stopped");
+			return;
+		}
+		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
+		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
+			if (!check_cqe_seen(cqe))
+				ERROR("unexpected error CQE, TX stopped");
+			return;
+		}
+#endif /* NDEBUG */
+		++cq_ci;
+	} while (1);
+	if (unlikely(cqe == NULL))
+		return;
+	txq->wqe_pi = ntohs(cqe->wqe_counter);
+	ctrl = (volatile struct mlx5_wqe_ctrl *)
+		tx_mlx5_wqe(txq, txq->wqe_pi);
+	elts_tail = ctrl->ctrl3;
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
+	/* Free buffers. */
+	while (elts_free != elts_tail) {
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (likely(m->pool == pool)) {
+				free[blk_n++] = m;
+			} else {
+				if (likely(pool != NULL))
+					rte_mempool_put_bulk(pool,
+							     (void *)free,
+							     blk_n);
+				free[0] = m;
+				pool = m->pool;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(pool, (void *)free, blk_n);
+#ifndef NDEBUG
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
+		memset(&(*txq->elts)[elts_free & elts_m],
+		       0x66,
+		       sizeof((*txq->elts)[elts_free & elts_m]));
+		++elts_free;
+	}
+#endif
+	txq->cq_ci = cq_ci;
+	txq->elts_tail = elts_tail;
+	/* Update the consumer index. */
+	rte_wmb();
+	*txq->cq_db = htonl(cq_ci);
+}
+
+/**
+ * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
+ * the cloned mbuf is allocated is returned instead.
+ *
+ * @param buf
+ *   Pointer to mbuf.
+ *
+ * @return
+ *   Memory pool where data is located for given mbuf.
+ */
+static struct rte_mempool *
+mlx5_tx_mb2mp(struct rte_mbuf *buf)
+{
+	if (unlikely(RTE_MBUF_INDIRECT(buf)))
+		return rte_mbuf_from_indirect(buf)->pool;
+	return buf->pool;
+}
+
+/**
+ * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
+ * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
+ * remove an entry first.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param[in] mp
+ *   Memory Pool for which a Memory Region lkey must be returned.
+ *
+ * @return
+ *   mr->lkey on success, (uint32_t)-1 on failure.
+ */
+static __rte_always_inline uint32_t
+mlx5_tx_mb2mr(struct txq *txq, struct rte_mbuf *mb)
+{
+	uint16_t i = txq->mr_cache_idx;
+	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
+
+	assert(i < RTE_DIM(txq->mp2mr));
+	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
+		return txq->mp2mr[i].lkey;
+	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
+		if (unlikely(txq->mp2mr[i].mr == NULL)) {
+			/* Unknown MP, add a new MR for it. */
+			break;
+		}
+		if (txq->mp2mr[i].start <= addr &&
+		    txq->mp2mr[i].end >= addr) {
+			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
+			assert(htonl(txq->mp2mr[i].mr->lkey) ==
+			       txq->mp2mr[i].lkey);
+			txq->mr_cache_idx = i;
+			return txq->mp2mr[i].lkey;
+		}
+	}
+	txq->mr_cache_idx = 0;
+	return txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i);
+}
+
+/**
+ * Ring TX queue doorbell.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param wqe
+ *   Pointer to the last WQE posted in the NIC.
+ */
+static __rte_always_inline void
+mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
+{
+	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
+	volatile uint64_t *src = ((volatile uint64_t *)wqe);
+
+	rte_wmb();
+	*txq->qp_db = htonl(txq->wqe_ci);
+	/* Ensure ordering between DB record and BF copy. */
+	rte_wmb();
+	*dst = *src;
+}
+
 #endif /* RTE_PMD_MLX5_RXTX_H_ */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.c b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
new file mode 100644
index 000000000..4595f7a33
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
@@ -0,0 +1,1378 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <smmintrin.h>
+
+/* Verbs header. */
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5_hw.h>
+#include <infiniband/arch.h>
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+#ifndef __INTEL_COMPILER
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#endif
+
+/**
+ * Fill in buffer descriptors in a multi-packet send descriptor.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param dseg
+ *   Pointer to buffer descriptor to be writen.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param n
+ *   Number of packets to be filled.
+ */
+static inline void
+txq_wr_dseg_v(struct txq *txq, __m128i *dseg,
+	      struct rte_mbuf **pkts, unsigned int n)
+{
+	unsigned int pos;
+	uintptr_t addr;
+	const __m128i shuf_mask_dseg =
+		_mm_set_epi8(8,  9, 10, 11, /* addr, bswap64 */
+			    12, 13, 14, 15,
+			     7,  6,  5,  4, /* lkey */
+			     0,  1,  2,  3  /* length, bswap32 */);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t tx_byte = 0;
+#endif
+
+	for (pos = 0; pos < n; ++pos, ++dseg) {
+		__m128i desc;
+		struct rte_mbuf *pkt = pkts[pos];
+
+		addr = rte_pktmbuf_mtod(pkt, uintptr_t);
+		desc = _mm_set_epi32(addr >> 32,
+				     addr,
+				     mlx5_tx_mb2mr(txq, pkt),
+				     DATA_LEN(pkt));
+		desc = _mm_shuffle_epi8(desc, shuf_mask_dseg);
+		_mm_store_si128(dseg, desc);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		tx_byte += DATA_LEN(pkt);
+#endif
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.obytes += tx_byte;
+#endif
+}
+
+/**
+ * Count the number of continuous single segment packets. The first packet must
+ * be a single segment packet.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ *
+ * @return
+ *   Number of continuous single segment packets.
+ */
+static inline unsigned int
+txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	unsigned int pos;
+
+	if (!pkts_n)
+		return 0;
+	assert(NB_SEGS(pkts[0]) == 1);
+	/* Count the number of continuous single segment packets. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if (NB_SEGS(pkts[pos]) > 1)
+			break;
+	return pos;
+}
+
+/**
+ * Count the number of packets having same ol_flags and calculate cs_flags.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ * @param cs_flags
+ *   Pointer of flags to be returned.
+ *
+ * @return
+ *   Number of packets having same ol_flags.
+ */
+static inline unsigned int
+txq_calc_offload(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+		 uint8_t *cs_flags)
+{
+	unsigned int pos;
+	const uint64_t ol_mask =
+		PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
+		PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
+		PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
+
+	if (!pkts_n)
+		return 0;
+	/* Count the number of packets having same ol_flags. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
+			break;
+	/* Should open another MPW session for the rest. */
+	if (pkts[0]->ol_flags &
+	    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+		const uint64_t is_tunneled =
+			pkts[0]->ol_flags &
+			(PKT_TX_TUNNEL_GRE |
+			 PKT_TX_TUNNEL_VXLAN);
+
+		if (is_tunneled && txq->tunnel_en) {
+			*cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
+				    MLX5_ETH_WQE_L4_INNER_CSUM;
+			if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+				*cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+		} else {
+			*cs_flags = MLX5_ETH_WQE_L3_CSUM |
+				    MLX5_ETH_WQE_L4_CSUM;
+		}
+	}
+	return pos;
+}
+
+/**
+ * Send multi-segmented packets until it encounters a single segment packet in
+ * the pkts list.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static uint16_t
+txq_scatter_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n;
+	volatile struct mlx5_wqe *wqe = NULL;
+
+	assert(elts_n > pkts_n);
+	mlx5_tx_complete(txq);
+	if (unlikely(!pkts_n))
+		return 0;
+	for (n = 0; n < pkts_n; ++n) {
+		struct rte_mbuf *buf = pkts[n];
+		unsigned int segs_n = buf->nb_segs;
+		unsigned int ds = nb_dword_in_hdr;
+		unsigned int len = PKT_LEN(buf);
+		uint16_t wqe_ci = txq->wqe_ci;
+		const __m128i shuf_mask_ctrl =
+			_mm_set_epi8(15, 14, 13, 12,
+				      8,  9, 10, 11, /* bswap32 */
+				      4,  5,  6,  7, /* bswap32 */
+				      0,  1,  2,  3  /* bswap32 */);
+		uint8_t cs_flags = 0;
+		uint16_t max_elts;
+		uint16_t max_wqe;
+		__m128i *t_wqe, *dseg;
+		__m128i ctrl;
+
+		assert(segs_n);
+		max_elts = elts_n - (elts_head - txq->elts_tail);
+		max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi);
+		/*
+		 * A MPW session consumes 2 WQEs at most to
+		 * include MLX5_MPW_DSEG_MAX pointers.
+		 */
+		if (segs_n == 1 ||
+		    max_elts < segs_n || max_wqe < 2)
+			break;
+		wqe = &((volatile struct mlx5_wqe64 *)
+			 txq->wqes)[wqe_ci & wq_mask].hdr;
+		if (buf->ol_flags &
+		     (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+			const uint64_t is_tunneled = buf->ol_flags &
+						      (PKT_TX_TUNNEL_GRE |
+						       PKT_TX_TUNNEL_VXLAN);
+
+			if (is_tunneled && txq->tunnel_en) {
+				cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
+					   MLX5_ETH_WQE_L4_INNER_CSUM;
+				if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+					cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+			} else {
+				cs_flags = MLX5_ETH_WQE_L3_CSUM |
+					   MLX5_ETH_WQE_L4_CSUM;
+			}
+		}
+		/* Title WQEBB pointer. */
+		t_wqe = (__m128i *)wqe;
+		dseg = (__m128i *)(wqe + 1);
+		do {
+			if (!(ds++ % nb_dword_per_wqebb)) {
+				dseg = (__m128i *)
+					&((volatile struct mlx5_wqe64 *)
+					   txq->wqes)[++wqe_ci & wq_mask];
+			}
+			txq_wr_dseg_v(txq, dseg++, &buf, 1);
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			buf = buf->next;
+		} while (--segs_n);
+		if (ds % nb_dword_per_wqebb)
+			++wqe_ci;
+		/* Fill CTRL in the header. */
+		ctrl = _mm_set_epi32(0, 0, txq->qp_num_8s | ds,
+				     MLX5_OPC_MOD_MPW << 24 |
+				     txq->wqe_ci << 8 | MLX5_OPCODE_TSO);
+		ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+		_mm_store_si128(t_wqe, ctrl);
+		/* Fill ESEG in the header. */
+		_mm_store_si128(t_wqe + 1,
+				_mm_set_epi16(0, 0, 0, 0,
+					      htons(len), cs_flags,
+					      0, 0));
+		txq->wqe_ci = wqe_ci;
+	}
+	if (!n)
+		return 0;
+	txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
+	txq->elts_head = elts_head;
+	if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+		wqe->ctrl[2] = htonl(8);
+		wqe->ctrl[3] = txq->elts_head;
+		txq->elts_comp = 0;
+		++txq->cq_pi;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += n;
+#endif
+	mlx5_tx_dbrec(txq, wqe);
+	return n;
+}
+
+/**
+ * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
+ * it returns to make it processed by txq_scatter_v(). All the packets in
+ * the pkts list should be single segment packets having same offload flags.
+ * This must be checked by txq_check_multiseg() and txq_calc_offload().
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST).
+ * @param cs_flags
+ *   Checksum offload flags to be written in the descriptor.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static inline uint16_t
+txq_burst_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+	    uint8_t cs_flags)
+{
+	struct rte_mbuf **elts;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n = 0;
+	unsigned int pos;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	uint32_t comp_req = 0;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	uint16_t wq_idx = txq->wqe_ci & wq_mask;
+	volatile struct mlx5_wqe64 *wq =
+		&((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx];
+	volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq;
+	const __m128i shuf_mask_ctrl =
+		_mm_set_epi8(15, 14, 13, 12,
+			      8,  9, 10, 11, /* bswap32 */
+			      4,  5,  6,  7, /* bswap32 */
+			      0,  1,  2,  3  /* bswap32 */);
+	__m128i *t_wqe, *dseg;
+	__m128i ctrl;
+
+	/* Make sure all packets can fit into a single WQE. */
+	assert(elts_n > pkts_n);
+	mlx5_tx_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
+	if (unlikely(!pkts_n))
+		return 0;
+	elts = &(*txq->elts)[elts_head & elts_m];
+	/* Loop for available tailroom first. */
+	n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n);
+	for (pos = 0; pos < (n & -2); pos += 2)
+		_mm_storeu_si128((__m128i *)&elts[pos],
+				 _mm_loadu_si128((__m128i *)&pkts[pos]));
+	if (n & 1)
+		elts[pos] = pkts[pos];
+	/* Check if it crosses the end of the queue. */
+	if (unlikely(n < pkts_n)) {
+		elts = &(*txq->elts)[0];
+		for (pos = 0; pos < pkts_n - n; ++pos)
+			elts[pos] = pkts[n + pos];
+	}
+	txq->elts_head += pkts_n;
+	/* Save title WQEBB pointer. */
+	t_wqe = (__m128i *)wqe;
+	dseg = (__m128i *)(wqe + 1);
+	/* Calculate the number of entries to the end. */
+	n = RTE_MIN(
+		(wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr,
+		pkts_n);
+	/* Fill DSEGs. */
+	txq_wr_dseg_v(txq, dseg, pkts, n);
+	/* Check if it crosses the end of the queue. */
+	if (n < pkts_n) {
+		dseg = (__m128i *)txq->wqes;
+		txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n);
+	}
+	if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
+		txq->elts_comp += pkts_n;
+	} else {
+		/* Request a completion. */
+		txq->elts_comp = 0;
+		++txq->cq_pi;
+		comp_req = 8;
+	}
+	/* Fill CTRL in the header. */
+	ctrl = _mm_set_epi32(txq->elts_head, comp_req,
+			     txq->qp_num_8s | (pkts_n + 2),
+			     MLX5_OPC_MOD_ENHANCED_MPSW << 24 |
+				txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW);
+	ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+	_mm_store_si128(t_wqe, ctrl);
+	/* Fill ESEG in the header. */
+	_mm_store_si128(t_wqe + 1,
+			_mm_set_epi8(0, 0, 0, 0,
+				     0, 0, 0, 0,
+				     0, 0, 0, cs_flags,
+				     0, 0, 0, 0));
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += pkts_n;
+#endif
+	txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) /
+		       nb_dword_per_wqebb;
+	/* Ring QP doorbell. */
+	mlx5_tx_dbrec(txq, wqe);
+	return pkts_n;
+}
+
+/**
+ * DPDK callback for vectorized TX.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
+		      uint16_t pkts_n)
+{
+	struct txq *txq = (struct txq *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint16_t n;
+		uint16_t ret;
+
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, 0);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * DPDK callback for vectorized TX with multi-seg packets and offload.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct txq *txq = (struct txq *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint8_t cs_flags = 0;
+		uint16_t n;
+		uint16_t ret;
+
+		/* Transmit multi-seg packets in the head of pkts list. */
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) &&
+		    NB_SEGS(pkts[nb_tx]) > 1)
+			nb_tx += txq_scatter_v(txq,
+					       &pkts[nb_tx],
+					       pkts_n - nb_tx);
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS))
+			n = txq_check_multiseg(&pkts[nb_tx], n);
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
+			n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * Store free buffers to RX SW ring.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be stored.
+ * @param pkts_n
+ *   Number of packets to be stored.
+ */
+static inline void
+rxq_copy_mbuf_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t n)
+{
+	const uint16_t q_mask = (1 << rxq->elts_n) - 1;
+	struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask];
+	unsigned int pos;
+	uint16_t p = n & -2;
+
+	for (pos = 0; pos < p; pos += 2) {
+		__m128i mbp;
+
+		mbp = _mm_loadu_si128((__m128i *)&elts[pos]);
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp);
+	}
+	if (n & 1)
+		pkts[pos] = elts[pos];
+}
+
+/**
+ * Replenish buffers for RX in bulk.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param n
+ *   Number of buffers to be replenished.
+ */
+static inline void
+rxq_replenish_bulk_mbuf(struct rxq *rxq, uint16_t n)
+{
+	const uint16_t q_n = 1 << rxq->elts_n;
+	const uint16_t q_mask = q_n - 1;
+	const uint16_t elts_idx = rxq->rq_ci & q_mask;
+	struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
+	volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx];
+	unsigned int i;
+
+	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH);
+	assert(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
+	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
+	if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
+		rxq->stats.rx_nombuf += n;
+		return;
+	}
+	for (i = 0; i < n; ++i)
+		wq[i].addr = htonll(rte_pktmbuf_mtod(elts[i], uintptr_t));
+	rxq->rq_ci += n;
+	rte_wmb();
+	*rxq->rq_db = htonl(rxq->rq_ci);
+}
+
+/**
+ * Decompress a compressed completion and fill in mbufs in RX SW ring with data
+ * extracted from the title completion descriptor.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cq
+ *   Pointer to completion array having a compressed completion at first.
+ * @param elts
+ *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
+ *   the title completion descriptor to be copied to the rest of mbufs.
+ */
+static inline void
+rxq_cq_decompress_v(struct rxq *rxq,
+		    volatile struct mlx5_cqe *cq,
+		    struct rte_mbuf **elts)
+{
+	volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + 1);
+	struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */
+	unsigned int pos;
+	unsigned int i;
+	unsigned int inv = 0;
+	/* Mask to shuffle from extracted mini CQE to mbuf. */
+	const __m128i shuf_mask1 =
+		_mm_set_epi8(0,  1,  2,  3, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			     6,  7,         /* data_len, bswap16 */
+			    -1, -1,  6,  7, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	const __m128i shuf_mask2 =
+		_mm_set_epi8(8,  9, 10, 11, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			    14, 15,         /* data_len, bswap16 */
+			    -1, -1, 14, 15, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	/* Restore the compressed count. Must be 16 bits. */
+	const uint16_t mcqe_n = t_pkt->data_len +
+				(rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i rearm =
+		_mm_loadu_si128((__m128i *)&t_pkt->rearm_data);
+	const __m128i rxdf =
+		_mm_loadu_si128((__m128i *)&t_pkt->rx_descriptor_fields1);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0, 0);
+	const uint32_t flow_tag = t_pkt->hash.fdir.hi;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     14, 15,  6,  7,
+			     10, 11,  2,  3);
+#endif
+
+	/* Compile time sanity check for this function. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+	/*
+	 * A. load mCQEs into a 128bit register.
+	 * B. store rearm data to mbuf.
+	 * C. combine data from mCQEs with rx_descriptor_fields1.
+	 * D. store rx_descriptor_fields1.
+	 * E. store flow tag (rte_flow mark).
+	 */
+	for (pos = 0; pos < mcqe_n; ) {
+		__m128i mcqe1, mcqe2;
+		__m128i rxdf1, rxdf2;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt, invalid_mask;
+#endif
+
+		if (!(pos & 0x7) && pos + 8 < mcqe_n)
+			rte_prefetch0((void *)(cq + pos + 8));
+		/* A.1 load mCQEs into a 128bit register. */
+		mcqe1 = _mm_loadu_si128((__m128i *)&mcq[pos % 8]);
+		mcqe2 = _mm_loadu_si128((__m128i *)&mcq[pos % 8 + 2]);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 1]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe1, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe1, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 1]->rx_descriptor_fields1,
+				 rxdf2);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos + 2]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 3]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe2, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe2, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 2]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 3]->rx_descriptor_fields1,
+				 rxdf2);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		invalid_mask = zero | ((mcqe_n - pos) * sizeof(uint16_t) * 8);
+		invalid_mask = _mm_sll_epi64(ones, invalid_mask);
+		mcqe1 = _mm_srli_si128(mcqe1, 4);
+		byte_cnt = _mm_blend_epi16(mcqe1, mcqe2, 0xcc);
+		byte_cnt = _mm_shuffle_epi8(byte_cnt, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		if (rxq->mark) {
+			/* E.1 store flow tag (rte_flow mark). */
+			elts[pos]->hash.fdir.hi = flow_tag;
+			elts[pos + 1]->hash.fdir.hi = flow_tag;
+			elts[pos + 2]->hash.fdir.hi = flow_tag;
+			elts[pos + 3]->hash.fdir.hi = flow_tag;
+		}
+		pos += MLX5_VPMD_DESCS_PER_LOOP;
+		/* Move to next CQE and invalidate consumed CQEs. */
+		if (!(pos & 0x7) && pos < mcqe_n) {
+			mcq = (void *)(cq + pos);
+			for (i = 0; i < 8; ++i)
+				cq[inv++].op_own = MLX5_CQE_INVALIDATE;
+		}
+	}
+	/* Invalidate the rest of CQEs. */
+	for (; inv < mcqe_n; ++inv)
+		cq[inv].op_own = MLX5_CQE_INVALIDATE;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += mcqe_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	rxq->cq_ci += mcqe_n;
+}
+
+/**
+ * Calculate packet type and offload flag for mbuf and store it.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cqes[4]
+ *   Array of four 16bytes completions extracted from the original completion
+ *   descriptor.
+ * @param op_err
+ *   Opcode vector having responder error status. Each field is 4B.
+ * @param pkts
+ *   Pointer to array of packets to be filled.
+ */
+static inline void
+rxq_cq_to_ptype_oflags_v(struct rxq *rxq, __m128i cqes[4], __m128i op_err,
+			 struct rte_mbuf **pkts)
+{
+	__m128i pinfo0, pinfo1;
+	__m128i pinfo, ptype;
+	__m128i ol_flags = _mm_set1_epi32(rxq->rss_hash * PKT_RX_RSS_HASH);
+	__m128i cv_flags;
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ptype_mask =
+		_mm_set_epi32(0xd06, 0xd06, 0xd06, 0xd06);
+	const __m128i ptype_ol_mask =
+		_mm_set_epi32(0x106, 0x106, 0x106, 0x106);
+	const __m128i pinfo_mask =
+		_mm_set_epi32(0x3, 0x3, 0x3, 0x3);
+	const __m128i cv_flag_sel =
+		_mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0,
+			     (uint8_t)((PKT_RX_IP_CKSUM_GOOD |
+					PKT_RX_L4_CKSUM_GOOD) >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1),
+			     (uint8_t)(PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED),
+			     0);
+	const __m128i cv_mask =
+		_mm_set_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED);
+	const __m128i mbuf_init =
+		_mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer);
+	__m128i rearm0, rearm1, rearm2, rearm3;
+
+	/* Extract pkt_info field. */
+	pinfo0 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+	pinfo = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	/* Extract hdr_type_etc field. */
+	pinfo0 = _mm_unpackhi_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpackhi_epi32(cqes[2], cqes[3]);
+	ptype = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	if (rxq->mark) {
+		const __m128i pinfo_ft_mask =
+			_mm_set_epi32(0xffffff00, 0xffffff00,
+				      0xffffff00, 0xffffff00);
+		const __m128i fdir_flags = _mm_set1_epi32(PKT_RX_FDIR);
+		const __m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID);
+		__m128i flow_tag, invalid_mask;
+
+		flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask);
+		/* Check if flow tag is non-zero then set PKT_RX_FDIR. */
+		invalid_mask = _mm_cmpeq_epi32(flow_tag, zero);
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(invalid_mask,
+							 fdir_flags));
+		/* Mask out invalid entries. */
+		flow_tag = _mm_andnot_si128(invalid_mask, flow_tag);
+		/* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(
+						_mm_cmpeq_epi32(flow_tag,
+								pinfo_ft_mask),
+						fdir_id_flags));
+	}
+	/*
+	 * Merge the two fields to generate the following:
+	 * bit[1]  = l3_ok,    bit[2]     = l4_ok
+	 * bit[8]  = cv,       bit[11:10] = l3_hdr_type
+	 * bit[12] = tunneled, bit[13]    = outer_l3_type
+	 */
+	ptype = _mm_and_si128(ptype, ptype_mask);
+	pinfo = _mm_and_si128(pinfo, pinfo_mask);
+	pinfo = _mm_slli_epi32(pinfo, 12);
+	ptype = _mm_or_si128(ptype, pinfo);
+	ptype = _mm_srli_epi32(ptype, 10);
+	ptype = _mm_packs_epi32(ptype, zero);
+	/* Errored packets will have RTE_PTYPE_ALL_MASK. */
+	op_err = _mm_srli_epi16(op_err, 12);
+	ptype = _mm_or_si128(ptype, op_err);
+	pkts[0]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 0)];
+	pkts[1]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 2)];
+	pkts[2]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 4)];
+	pkts[3]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 6)];
+	/* Fill flags for checksum and VLAN. */
+	pinfo = _mm_and_si128(pinfo, ptype_ol_mask);
+	pinfo = _mm_shuffle_epi8(cv_flag_sel, pinfo);
+	/* Locate checksum flags at byte[2:1] and merge with VLAN flags. */
+	cv_flags = _mm_slli_epi32(pinfo, 9);
+	cv_flags = _mm_or_si128(pinfo, cv_flags);
+	/* Move back flags to start from byte[0]. */
+	cv_flags = _mm_srli_epi32(cv_flags, 8);
+	/* Mask out garbage bits. */
+	cv_flags = _mm_and_si128(cv_flags, cv_mask);
+	/* Merge to ol_flags. */
+	ol_flags = _mm_or_si128(ol_flags, cv_flags);
+	/* Merge mbuf_init and ol_flags. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+			 offsetof(struct rte_mbuf, rearm_data) + 8);
+	rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 8), 0x30);
+	rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 4), 0x30);
+	rearm2 = _mm_blend_epi16(mbuf_init, ol_flags, 0x30);
+	rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(ol_flags, 4), 0x30);
+	/* Write 8B rearm_data and 8B ol_flags. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+			 RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+	_mm_store_si128((__m128i *)&pkts[0]->rearm_data, rearm0);
+	_mm_store_si128((__m128i *)&pkts[1]->rearm_data, rearm1);
+	_mm_store_si128((__m128i *)&pkts[2]->rearm_data, rearm2);
+	_mm_store_si128((__m128i *)&pkts[3]->rearm_data, rearm3);
+}
+
+/**
+ * Skip error packets.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+static uint16_t
+rxq_handle_pending_error(struct rxq *rxq, struct rte_mbuf **pkts,
+			 uint16_t pkts_n)
+{
+	uint16_t n = 0;
+	unsigned int i;
+
+	for (i = 0; i < pkts_n; ++i) {
+		struct rte_mbuf *pkt = pkts[i];
+
+		if (pkt->packet_type == RTE_PTYPE_ALL_MASK)
+			rte_pktmbuf_free_seg(pkt);
+		else
+			pkts[n++] = pkt;
+	}
+	rxq->stats.idropped += (pkts_n - n);
+	rxq->pending_err = 0;
+	return n;
+}
+
+/**
+ * Receive burst of packets. An errored completion also consumes a mbuf, but the
+ * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed
+ * before returning to application.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets received including errors (<= pkts_n).
+ */
+static inline uint16_t
+rxq_burst_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	const uint16_t q_n = 1 << rxq->cqe_n;
+	const uint16_t q_mask = q_n - 1;
+	volatile struct mlx5_cqe *cq;
+	struct rte_mbuf **elts;
+	unsigned int pos;
+	uint64_t n;
+	uint16_t repl_n;
+	uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
+	uint16_t nocmp_n = 0;
+	uint16_t rcvd_pkt = 0;
+	unsigned int cq_idx = rxq->cq_ci & q_mask;
+	unsigned int elts_idx;
+	unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1));
+	const __m128i owner_check =
+		_mm_set_epi64x(0x0100000001000000LL, 0x0100000001000000LL);
+	const __m128i opcode_check =
+		_mm_set_epi64x(0xf0000000f0000000LL, 0xf0000000f0000000LL);
+	const __m128i format_check =
+		_mm_set_epi64x(0x0c0000000c000000LL, 0x0c0000000c000000LL);
+	const __m128i resp_err_check =
+		_mm_set_epi64x(0xe0000000e0000000LL, 0xe0000000e0000000LL);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     12, 13,  8,  9,
+			      4,  5,  0,  1);
+#endif
+	/* Mask to shuffle from extracted CQE to mbuf. */
+	const __m128i shuf_mask =
+		_mm_set_epi8(-1,  3,  2,  1, /* fdir.hi */
+			     12, 13, 14, 15, /* rss, bswap32 */
+			     10, 11,         /* vlan_tci, bswap16 */
+			      4,  5,         /* data_len, bswap16 */
+			     -1, -1,         /* zero out 2nd half of pkt_len */
+			      4,  5          /* pkt_len, bswap16 */);
+	/* Mask to blend from the last Qword to the first DQword. */
+	const __m128i blend_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			      0,  0,  0,  0,
+			      0,  0,  0, -1);
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0);
+
+	/* Compile time sanity check for this function. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, pkt_info) != 0);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rx_hash_res) !=
+			 offsetof(struct mlx5_cqe, pkt_info) + 12);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd1) +
+			  sizeof(((struct mlx5_cqe *)0)->rsvd1) !=
+			 offsetof(struct mlx5_cqe, hdr_type_etc));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, vlan_info) !=
+			 offsetof(struct mlx5_cqe, hdr_type_etc) + 2);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd2) +
+			  sizeof(((struct mlx5_cqe *)0)->rsvd2) !=
+			 offsetof(struct mlx5_cqe, byte_cnt));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, sop_drop_qpn) !=
+			 RTE_ALIGN(offsetof(struct mlx5_cqe, sop_drop_qpn), 8));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, op_own) !=
+			 offsetof(struct mlx5_cqe, sop_drop_qpn) + 7);
+	assert(rxq->sges_n == 0);
+	assert(rxq->cqe_n == rxq->elts_n);
+	cq = &(*rxq->cqes)[cq_idx];
+	rte_prefetch0(cq);
+	rte_prefetch0(cq + 1);
+	rte_prefetch0(cq + 2);
+	rte_prefetch0(cq + 3);
+	pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
+	/*
+	 * Order of indexes:
+	 *   rq_ci >= cq_ci >= rq_pi
+	 * Definition of indexes:
+	 *   rq_ci - cq_ci := # of buffers owned by HW (posted).
+	 *   cq_ci - rq_pi := # of buffers not returned to app (decompressed).
+	 *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
+	 */
+	repl_n = q_n - (rxq->rq_ci - rxq->rq_pi);
+	if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH)
+		rxq_replenish_bulk_mbuf(rxq, repl_n);
+	/* See if there're unreturned mbufs from compressed CQE. */
+	rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
+	if (rcvd_pkt > 0) {
+		rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
+		rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt);
+		rxq->rq_pi += rcvd_pkt;
+		pkts += rcvd_pkt;
+	}
+	elts_idx = rxq->rq_pi & q_mask;
+	elts = &(*rxq->elts)[elts_idx];
+	/* Not to overflow pkts array. */
+	pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
+	if (!pkts_n)
+		return rcvd_pkt;
+	/* At this point, there shouldn't be any remained packets. */
+	assert(rxq->rq_pi == rxq->cq_ci);
+	/*
+	 * A. load first Qword (8bytes) in one loop.
+	 * B. copy 4 mbuf pointers from elts ring to returing pkts.
+	 * C. load remained CQE data and extract necessary fields.
+	 *    Final 16bytes cqes[] extracted from original 64bytes CQE has the
+	 *    following structure:
+	 *        struct {
+	 *          uint8_t  pkt_info;
+	 *          uint8_t  flow_tag[3];
+	 *          uint16_t byte_cnt;
+	 *          uint8_t  rsvd4;
+	 *          uint8_t  op_own;
+	 *          uint16_t hdr_type_etc;
+	 *          uint16_t vlan_info;
+	 *          uint32_t rx_has_res;
+	 *        } c;
+	 * D. fill in mbuf.
+	 * E. get valid CQEs.
+	 * F. find compressed CQE.
+	 */
+	for (pos = 0;
+	     pos < pkts_n;
+	     pos += MLX5_VPMD_DESCS_PER_LOOP) {
+		__m128i cqes[MLX5_VPMD_DESCS_PER_LOOP];
+		__m128i cqe_tmp1, cqe_tmp2;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
+		__m128i op_own, op_own_tmp1, op_own_tmp2;
+		__m128i opcode, owner_mask, invalid_mask;
+		__m128i comp_mask;
+		__m128i mask;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt;
+#endif
+		__m128i mbp1, mbp2;
+		__m128i p = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0);
+		unsigned int p1, p2, p3;
+
+		/* Prefetch next 4 CQEs. */
+		if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) {
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]);
+		}
+		/* A.0 do not cross the end of CQ. */
+		mask = zero | ((pkts_n - pos) * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		p = _mm_andnot_si128(mask, p);
+		/* A.1 load cqes. */
+		p3 = _mm_extract_epi16(p, 3);
+		cqes[3] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p3].sop_drop_qpn);
+		rte_compiler_barrier();
+		p2 = _mm_extract_epi16(p, 2);
+		cqes[2] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p2].sop_drop_qpn);
+		rte_compiler_barrier();
+		/* B.1 load mbuf pointers. */
+		mbp1 = _mm_loadu_si128((__m128i *)&elts[pos]);
+		mbp2 = _mm_loadu_si128((__m128i *)&elts[pos + 2]);
+		/* A.1 load a block having op_own. */
+		p1 = _mm_extract_epi16(p, 1);
+		cqes[1] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p1].sop_drop_qpn);
+		rte_compiler_barrier();
+		cqes[0] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos].sop_drop_qpn);
+		/* B.2 copy mbuf pointers. */
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp1);
+		_mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2);
+		rte_compiler_barrier();
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]);
+		cqes[3] = _mm_blendv_epi8(cqes[3], cqe_tmp2, blend_mask);
+		cqes[2] = _mm_blendv_epi8(cqes[2], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p3].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos + p2].rsvd1[3]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x30);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p3].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos + p2].rsvd2[10]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x04);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb3 = _mm_shuffle_epi8(cqes[3], shuf_mask);
+		pkt_mb2 = _mm_shuffle_epi8(cqes[2], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb3 = _mm_sub_epi16(pkt_mb3, crc_adj);
+		pkt_mb2 = _mm_sub_epi16(pkt_mb2, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb3 = _mm_add_epi32(pkt_mb3, flow_mark_adj);
+		pkt_mb2 = _mm_add_epi32(pkt_mb2, flow_mark_adj);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 3]->pkt_len, pkt_mb3);
+		_mm_storeu_si128((void *)&pkts[pos + 2]->pkt_len, pkt_mb2);
+		/* E.1 extract op_own field. */
+		op_own_tmp2 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p1]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos]);
+		cqes[1] = _mm_blendv_epi8(cqes[1], cqe_tmp2, blend_mask);
+		cqes[0] = _mm_blendv_epi8(cqes[0], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p1].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos].rsvd1[3]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x30);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p1].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos].rsvd2[10]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x04);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb1 = _mm_shuffle_epi8(cqes[1], shuf_mask);
+		pkt_mb0 = _mm_shuffle_epi8(cqes[0], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb1 = _mm_sub_epi16(pkt_mb1, crc_adj);
+		pkt_mb0 = _mm_sub_epi16(pkt_mb0, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb1 = _mm_add_epi32(pkt_mb1, flow_mark_adj);
+		pkt_mb0 = _mm_add_epi32(pkt_mb0, flow_mark_adj);
+		/* E.1 extract op_own byte. */
+		op_own_tmp1 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+		op_own = _mm_unpackhi_epi64(op_own_tmp1, op_own_tmp2);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 1]->pkt_len, pkt_mb1);
+		_mm_storeu_si128((void *)&pkts[pos]->pkt_len, pkt_mb0);
+		/* E.2 flip owner bit to mark CQEs from last round. */
+		owner_mask = _mm_and_si128(op_own, owner_check);
+		if (ownership)
+			owner_mask = _mm_xor_si128(owner_mask, owner_check);
+		owner_mask = _mm_cmpeq_epi32(owner_mask, owner_check);
+		owner_mask = _mm_packs_epi32(owner_mask, zero);
+		/* E.3 get mask for invalidated CQEs. */
+		opcode = _mm_and_si128(op_own, opcode_check);
+		invalid_mask = _mm_cmpeq_epi32(opcode_check, opcode);
+		invalid_mask = _mm_packs_epi32(invalid_mask, zero);
+		/* E.4 mask out beyond boundary. */
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* E.5 merge invalid_mask with invalid owner. */
+		invalid_mask = _mm_or_si128(invalid_mask, owner_mask);
+		/* F.1 find compressed CQE format. */
+		comp_mask = _mm_and_si128(op_own, format_check);
+		comp_mask = _mm_cmpeq_epi32(comp_mask, format_check);
+		comp_mask = _mm_packs_epi32(comp_mask, zero);
+		/* F.2 mask out invalid entries. */
+		comp_mask = _mm_andnot_si128(invalid_mask, comp_mask);
+		comp_idx = _mm_cvtsi128_si64(comp_mask);
+		/* F.3 get the first compressed CQE. */
+		comp_idx = comp_idx ?
+				__builtin_ctzll(comp_idx) /
+					(sizeof(uint16_t) * 8) :
+				MLX5_VPMD_DESCS_PER_LOOP;
+		/* E.6 mask out entries after the compressed CQE. */
+		mask = zero | (comp_idx * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* E.7 count non-compressed valid CQEs. */
+		n = _mm_cvtsi128_si64(invalid_mask);
+		n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) :
+			MLX5_VPMD_DESCS_PER_LOOP;
+		nocmp_n += n;
+		/* D.2 get the final invalid mask. */
+		mask = zero | (n * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* D.3 check error in opcode. */
+		opcode = _mm_cmpeq_epi32(resp_err_check, opcode);
+		opcode = _mm_packs_epi32(opcode, zero);
+		opcode = _mm_andnot_si128(invalid_mask, opcode);
+		/* D.4 mark if any error is set */
+		rxq->pending_err |= !!_mm_cvtsi128_si64(opcode);
+		/* D.5 fill in mbuf - rearm_data and packet_type. */
+		rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Add up received bytes count. */
+		byte_cnt = _mm_shuffle_epi8(op_own, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		/*
+		 * Break the loop unless more valid CQE is expected, or if
+		 * there's a compressed CQE.
+		 */
+		if (n != MLX5_VPMD_DESCS_PER_LOOP)
+			break;
+	}
+	/* If no new CQE seen, return without updating cq_db. */
+	if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP))
+		return rcvd_pkt;
+	/* Update the consumer indexes for non-compressed CQEs. */
+	assert(nocmp_n <= pkts_n);
+	rxq->cq_ci += nocmp_n;
+	rxq->rq_pi += nocmp_n;
+	rcvd_pkt += nocmp_n;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += nocmp_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	/* Decompress the last CQE if compressed. */
+	if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) {
+		assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
+		rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]);
+		/* Return more packets if needed. */
+		if (nocmp_n < pkts_n) {
+			uint16_t n = rxq->cq_ci - rxq->rq_pi;
+
+			n = RTE_MIN(n, pkts_n - nocmp_n);
+			rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n);
+			rxq->rq_pi += n;
+			rcvd_pkt += n;
+		}
+	}
+	rte_wmb();
+	*rxq->cq_db = htonl(rxq->cq_ci);
+	return rcvd_pkt;
+}
+
+/**
+ * DPDK callback for vectorized RX.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct rxq *rxq = dpdk_rxq;
+	uint16_t nb_rx;
+
+	nb_rx = rxq_burst_v(rxq, pkts, pkts_n);
+	if (unlikely(rxq->pending_err))
+		nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
+	return nb_rx;
+}
+
+/**
+ * Check Tx queue flags are set for raw vectorized Tx.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_raw_vec_tx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->txqs_n; ++i) {
+		struct txq *txq = (*priv->txqs)[i];
+
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) ||
+		    !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
+			break;
+	}
+	if (i != priv->txqs_n)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized TX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_vec_tx_support(struct priv *priv)
+{
+	if (priv->txqs_n > MLX5_VPMD_MIN_TXQS ||
+	    priv->mps != MLX5_MPW_ENHANCED ||
+	    priv->tso)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a RX queue can support vectorized RX.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+rxq_check_vec_support(struct rxq *rxq)
+{
+	if (rxq->sges_n != 0)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized RX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_vec_rx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+
+		if (rxq_check_vec_support(rxq) < 0)
+			break;
+	}
+	if (i != priv->rxqs_n)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Prepare for vectorized RX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+void
+priv_prep_vec_rx_function(struct priv *priv)
+{
+	uint16_t i;
+
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+		struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
+		const uint16_t desc = 1 << rxq->elts_n;
+		int j;
+
+		assert(rxq->elts_n == rxq->cqe_n);
+		/* Initialize default rearm_data for vPMD. */
+		mbuf_init->data_off = RTE_PKTMBUF_HEADROOM;
+		rte_mbuf_refcnt_set(mbuf_init, 1);
+		mbuf_init->nb_segs = 1;
+		mbuf_init->port = rxq->port_id;
+		/*
+		 * prevent compiler reordering:
+		 * rearm_data covers previous fields.
+		 */
+		rte_compiler_barrier();
+		rxq->mbuf_initializer =
+			*(uint64_t *)&mbuf_init->rearm_data;
+		/* Padding with a fake mbuf for vectorized Rx. */
+		for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j)
+			(*rxq->elts)[desc + j] = &rxq->fake_mbuf;
+		/* Mark that it need to be cleaned up for rxq_alloc_elts(). */
+		rxq->trim_elts = 1;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 03e23f58d..3ad6ac0c6 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -243,7 +243,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
 		ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set");
 		goto error;
 	}
-	(void)conf; /* Thresholds configuration (ignored). */
+	tmpl.txq.flags = conf->txq_flags;
 	assert(desc > MLX5_TX_COMP_THRESH);
 	tmpl.txq.elts_n = log2above(desc);
 	if (priv->mps == MLX5_MPW_ENHANCED)
-- 
2.11.0

Re: [dpdk-dev] [PATCH v3 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Yongseok Koh]

> On Jul 5, 2017, at 11:12 AM, Yongseok Koh <yskoh@mellanox.com> wrote:
> 
> To make vectorized burst routines enabled, it is required to run on x86_64
> architecture which can support at least SSE4.1. If all the conditions are
> met, the vectorized burst functions are enabled automatically. The decision
> is made individually on RX and TX. There's no PMD option to make a
> selection.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---

Hi Ferruh,

I forgot to amend subject/description of this patch. Shall I submit v4? Or, can
you change it when you merge it, if the whole series of v3 is acked (by Nelio)?

If this patch can be accepted without submitting v4, then please amend the
commit message to:

------
net/mlx5: add vectorized Rx/Tx burst for x86

To make vectorized burst routines enabled, it is required to run on x86_64
architecture. If all the conditions are met, the vectorized burst functions
are enabled automatically. The decision is made individually on RX and TX.
There's no PMD option to make a selection

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>

------

This is because DPDK now has supporting SSE4.2 as a minimum requirement for x86
host and I also dropped the requirement in v3.

If I need to submit v4 to make the change, please let me know.

Thanks,
Yongseok

Re: [dpdk-dev] [PATCH v3 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Nélio Laranjeiro]

On Wed, Jul 05, 2017 at 11:12:28AM -0700, Yongseok Koh wrote:
>[...]
> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.c b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
> new file mode 100644
> index 000000000..4595f7a33
> --- /dev/null
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
> @@ -0,0 +1,1378 @@
>[...]
> +
> +#include <assert.h>
> +#include <stdint.h>
> +#include <string.h>
> +#include <stdlib.h>
> +#include <smmintrin.h>
> +
> +/* Verbs header. */
> +#include <infiniband/verbs.h>
> +#include <infiniband/mlx5_hw.h>
> +#include <infiniband/arch.h>

Pragma on Verbs must remain, their headers are still not clear for
pdentic compilation.

>[...]
> +/**
> + * Replenish buffers for RX in bulk.
> + *
> + * @param rxq
> + *   Pointer to RX queue structure.
> + * @param n
> + *   Number of buffers to be replenished.
> + */
> +static inline void
> +rxq_replenish_bulk_mbuf(struct rxq *rxq, uint16_t n)
> +{
> +	const uint16_t q_n = 1 << rxq->elts_n;
> +	const uint16_t q_mask = q_n - 1;
> +	const uint16_t elts_idx = rxq->rq_ci & q_mask;
> +	struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
> +	volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx];
> +	unsigned int i;
> +
> +	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH);
> +	assert(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
> +	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP);

Info: as the PMD is compiled in C11, you could use the static_assert
which verifies at compilation time the constants are correct.  
(I am not asking to change it now, we can make a compain to change
all this kind of assert at onces).

Unless the pragma which must remain on Verbs header:

Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

Great work.
Thanks,

-- 
Nélio Laranjeiro
6WIND

Re: [dpdk-dev] [PATCH v3 5/5] net/mlx5: add vectorized Rx/Tx burst for SSE4.1

[Ferruh Yigit]

On 7/5/2017 7:12 PM, Yongseok Koh wrote:
> To make vectorized burst routines enabled, it is required to run on x86_64
> architecture which can support at least SSE4.1. If all the conditions are
> met, the vectorized burst functions are enabled automatically. The decision
> is made individually on RX and TX. There's no PMD option to make a
> selection.
> 
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>

Hi Yongseok,

clang [2] is producing following build error [1], can you please check?

Thanks,
ferruh


[1]
.../dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.c:686:23: error: cannot
convert between vector values of different size ('__m128i' (vector of 2
'long long' values) and 'unsigned long')
                invalid_mask = zero | ((mcqe_n - pos) * sizeof(uint16_t)
* 8);
                               ~~~~ ^
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.../dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.c:1055:15: error: cannot
convert between vector values of different size ('__m128i' (vector of 2
'long long' values) and 'unsigned long')
                mask = zero | ((pkts_n - pos) * sizeof(uint16_t) * 8);
                       ~~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.../dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.c:1163:15: error: cannot
convert between vector values of different size ('__m128i' (vector of 2
'long long' values) and 'unsigned long')
                mask = zero | (comp_idx * sizeof(uint16_t) * 8);
                       ~~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.../dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.c:1172:15: error: cannot
convert between vector values of different size ('__m128i' (vector of 2
'long long' values) and 'unsigned long')
                mask = zero | (n * sizeof(uint16_t) * 8);
                       ~~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~
4 errors generated.

[2]
ICC is giving build error for same lines, I assume fixing clang will fix
the ICC,

[dpdk-dev] [PATCH v4 0/5] net/mlx5: add vectorized Rx/Tx burst for x86

[Yongseok Koh]

This is to introduce more efficient Rx/Tx burst functions using SIMD
instructions. Currently it is only supported by 64bit x86 having SSE4.1.

>From functional perspective, Rx burst function is equivalent to the
existing mlx5_rx_burst() except for scatter support, which will be added
soon. Tx burst function supports multi-segment packets and offload flags
unless it is disabled by txq_flags. However, disabling those features will
bring a little higher performance.

v4:
* Restore 'pragma' for PEDANTIC.
* Resolve compile error with clang.

v3:
* Remove requirement of SSE4.1 as DPDK now mandates SSE4.2 support at least.
* Bug fix in "net/mlx5: select Rx/Tx callbacks when starting device"
  - Need to re-select Rx burst func when chaning MTU size.
* Resolved an optimization issue of gcc-6 in rxq_burst_v()
  - Bit shift (<<) for 128b vector type is compiled differently. 'psllq' is
    needed instead of 'sal'.
* Minor changes to address what is mentioned by review.
  - Remove 'pragma' for PEDANTIC
  - Make mlx5_ptype_table global.
  - Change name of some inline funcs which also exist in mlx4 by the same name.
  - Fix comments and indentation/spacing.

v2:
* Streamline redundant conditional clauses in txq_complete().
* Remove the mempool pointer in txq->mp2mr structure.
* Fix indentation and spacing.

Yongseok Koh (5):
  net/mlx5: change indexing for Tx SW ring
  net/mlx5: free buffers in bulk on Tx completion
  net/mlx5: use buffer address for LKEY search
  net/mlx5: select Rx/Tx callbacks when starting device
  net/mlx5: add vectorized Rx/Tx burst for x86

 drivers/net/mlx5/Makefile            |    3 +
 drivers/net/mlx5/mlx5_defs.h         |   18 +
 drivers/net/mlx5/mlx5_ethdev.c       |   47 +-
 drivers/net/mlx5/mlx5_mr.c           |   17 +-
 drivers/net/mlx5/mlx5_rxq.c          |   57 +-
 drivers/net/mlx5/mlx5_rxtx.c         |  459 ++++-------
 drivers/net/mlx5/mlx5_rxtx.h         |  290 ++++++-
 drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1394 ++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_trigger.c      |    3 +
 drivers/net/mlx5/mlx5_txq.c          |   23 +-
 10 files changed, 1943 insertions(+), 368 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c

-- 
2.11.0

[dpdk-dev] [PATCH v4 1/5] net/mlx5: change indexing for Tx SW ring

[Yongseok Koh]

For Tx SW ring (txq->elts[]), indexes are kept and used in
txq->elts_head/tail. Because of this, one entry must always be left unused
and it also makes code complex. Changed to store counters instead of
indexes in order to make the code simpler and to reduce a few calculations.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 108 +++++++++++++++++--------------------------
 drivers/net/mlx5/mlx5_rxtx.h |   4 +-
 drivers/net/mlx5/mlx5_txq.c  |  16 +++----
 3 files changed, 52 insertions(+), 76 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 70314b393..43db06ad8 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -255,7 +255,8 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
 static inline void
 txq_complete(struct txq *txq)
 {
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	const unsigned int cqe_n = 1 << txq->cqe_n;
 	const unsigned int cqe_cnt = cqe_n - 1;
 	uint16_t elts_free = txq->elts_tail;
@@ -292,24 +293,23 @@ txq_complete(struct txq *txq)
 	ctrl = (volatile struct mlx5_wqe_ctrl *)
 		tx_mlx5_wqe(txq, txq->wqe_pi);
 	elts_tail = ctrl->ctrl3;
-	assert(elts_tail < (1 << txq->wqe_n));
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
 	/* Free buffers. */
 	while (elts_free != elts_tail) {
-		struct rte_mbuf *elt = (*txq->elts)[elts_free];
-		unsigned int elts_free_next =
-			(elts_free + 1) & (elts_n - 1);
-		struct rte_mbuf *elt_next = (*txq->elts)[elts_free_next];
+		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
+		struct rte_mbuf *elt_next =
+			(*txq->elts)[(elts_free + 1) & elts_m];
 
 #ifndef NDEBUG
 		/* Poisoning. */
-		memset(&(*txq->elts)[elts_free],
+		memset(&(*txq->elts)[elts_free & elts_m],
 		       0x66,
-		       sizeof((*txq->elts)[elts_free]));
+		       sizeof((*txq->elts)[elts_free & elts_m]));
 #endif
 		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
 		/* Only one segment needs to be freed. */
 		rte_pktmbuf_free_seg(elt);
-		elts_free = elts_free_next;
+		++elts_free;
 	}
 	txq->cq_ci = cq_ci;
 	txq->elts_tail = elts_tail;
@@ -409,12 +409,10 @@ int
 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
 {
 	struct txq *txq = tx_queue;
-	const unsigned int elts_n = 1 << txq->elts_n;
-	const unsigned int elts_cnt = elts_n - 1;
-	unsigned int used;
+	uint16_t used;
 
 	txq_complete(txq);
-	used = (txq->elts_head - txq->elts_tail) & elts_cnt;
+	used = txq->elts_head - txq->elts_tail;
 	if (offset < used)
 		return RTE_ETH_TX_DESC_FULL;
 	return RTE_ETH_TX_DESC_DONE;
@@ -488,11 +486,12 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
 	unsigned int k = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	unsigned int max_inline = txq->max_inline;
 	const unsigned int inline_en = !!max_inline && txq->inline_en;
 	uint16_t max_wqe;
@@ -509,9 +508,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(*pkts);
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
@@ -540,9 +537,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--segs_n;
 		if (unlikely(--max_wqe == 0))
 			break;
@@ -561,7 +558,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		if (length < (MLX5_WQE_DWORD_SIZE + 2))
 			break;
 		/* Update element. */
-		(*txq->elts)[elts_head] = buf;
+		(*txq->elts)[elts_head & elts_m] = buf;
 		/* Prefetch next buffer data. */
 		if (pkts_n - i > 1)
 			rte_prefetch0(
@@ -801,8 +798,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr,
 			naddr >> 32,
 		};
-		elts_head = (elts_head + 1) & (elts_n - 1);
-		(*txq->elts)[elts_head] = buf;
+		(*txq->elts)[++elts_head & elts_m] = buf;
 		++sg;
 		/* Advance counter only if all segs are successfully posted. */
 		if (sg < segs_n)
@@ -810,7 +806,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		else
 			j += sg;
 next_pkt:
-		elts_head = (elts_head + 1) & (elts_n - 1);
+		++elts_head;
 		++pkts;
 		++i;
 		/* Initialize known and common part of the WQE structure. */
@@ -853,7 +849,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Take a shortcut if nothing must be sent. */
 	if (unlikely((i + k) == 0))
 		return 0;
-	txq->elts_head = (txq->elts_head + i + j) & (elts_n - 1);
+	txq->elts_head += (i + j);
 	/* Check whether completion threshold has been reached. */
 	comp = txq->elts_comp + i + j + k;
 	if (comp >= MLX5_TX_COMP_THRESH) {
@@ -960,10 +956,11 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int comp;
 	struct mlx5_mpw mpw = {
@@ -977,15 +974,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
 		return 0;
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
 		uint32_t cs_flags = 0;
@@ -995,12 +989,12 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--pkts_n;
 		/* Should we enable HW CKSUM offload */
 		if (buf->ol_flags &
@@ -1036,9 +1030,8 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			volatile struct mlx5_wqe_data_seg *dseg;
 			uintptr_t addr;
 
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
 			assert(buf);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			dseg = mpw.data.dseg[mpw.pkts_n];
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
@@ -1046,7 +1039,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 				.addr = htonll(addr),
 			};
-			elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 			length += DATA_LEN(buf);
 #endif
@@ -1057,7 +1049,6 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		assert(length == mpw.len);
 		if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
 			mlx5_mpw_close(txq, &mpw);
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
@@ -1175,10 +1166,11 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int comp;
 	unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE;
@@ -1205,12 +1197,9 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
 	txq_complete(txq);
-	max = (elts_n - (elts_head - txq->elts_tail));
-	if (max > elts_n)
-		max -= elts_n;
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uintptr_t addr;
 		uint32_t length;
 		unsigned int segs_n = buf->nb_segs;
@@ -1221,12 +1210,12 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max < segs_n + 1)
+		if (max_elts < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
 			break;
-		max -= segs_n;
+		max_elts -= segs_n;
 		--pkts_n;
 		/*
 		 * Compute max_wqe in case less WQE were consumed in previous
@@ -1287,10 +1276,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 			do {
 				volatile struct mlx5_wqe_data_seg *dseg;
 
-				elts_head_next =
-					(elts_head + 1) & (elts_n - 1);
 				assert(buf);
-				(*txq->elts)[elts_head] = buf;
+				(*txq->elts)[elts_head++ & elts_m] = buf;
 				dseg = mpw.data.dseg[mpw.pkts_n];
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
@@ -1298,7 +1285,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 					.addr = htonll(addr),
 				};
-				elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 				length += DATA_LEN(buf);
 #endif
@@ -1315,9 +1301,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 			assert(mpw.state == MLX5_MPW_INL_STATE_OPENED);
 			assert(length <= inline_room);
 			assert(length == DATA_LEN(buf));
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			/* Maximum number of bytes before wrapping. */
 			max = ((((uintptr_t)(txq->wqes)) +
 				(1 << txq->wqe_n) *
@@ -1354,7 +1339,6 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				inline_room -= length;
 			}
 		}
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
@@ -1476,10 +1460,11 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 {
 	struct txq *txq = (struct txq *)dpdk_txq;
 	uint16_t elts_head = txq->elts_head;
-	const unsigned int elts_n = 1 << txq->elts_n;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
 	unsigned int i = 0;
 	unsigned int j = 0;
-	unsigned int max_elts;
+	uint16_t max_elts;
 	uint16_t max_wqe;
 	unsigned int max_inline = txq->max_inline * RTE_CACHE_LINE_SIZE;
 	unsigned int mpw_room = 0;
@@ -1494,8 +1479,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Start processing. */
 	txq_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
-	if (max_elts > elts_n)
-		max_elts -= elts_n;
 	/* A CQE slot must always be available. */
 	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
@@ -1503,7 +1486,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		return 0;
 	do {
 		struct rte_mbuf *buf = *(pkts++);
-		unsigned int elts_head_next;
 		uintptr_t addr;
 		uint64_t naddr;
 		unsigned int n;
@@ -1517,7 +1499,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * that one ring entry remains unused.
 		 */
 		assert(segs_n);
-		if (max_elts - j < segs_n + 1)
+		if (max_elts - j < segs_n)
 			break;
 		/* Do not bother with large packets MPW cannot handle. */
 		if (segs_n > MLX5_MPW_DSEG_MAX)
@@ -1601,10 +1583,8 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			do {
 				volatile struct mlx5_wqe_data_seg *dseg;
 
-				elts_head_next =
-					(elts_head + 1) & (elts_n - 1);
 				assert(buf);
-				(*txq->elts)[elts_head] = buf;
+				(*txq->elts)[elts_head++ & elts_m] = buf;
 				dseg = mpw.data.dseg[mpw.pkts_n];
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
@@ -1612,7 +1592,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
 					.addr = htonll(addr),
 				};
-				elts_head = elts_head_next;
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
 				length += DATA_LEN(buf);
 #endif
@@ -1663,7 +1642,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			/* No need to get completion as the entire packet is
 			 * copied to WQ. Free the buf right away.
 			 */
-			elts_head_next = elts_head;
 			rte_pktmbuf_free_seg(buf);
 			mpw_room -= (inl_pad + sizeof(inl_hdr) + length);
 			/* Add pad in the next packet if any. */
@@ -1686,8 +1664,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				dseg = (volatile void *)
 					((uintptr_t)mpw.data.raw +
 					 inl_pad);
-			elts_head_next = (elts_head + 1) & (elts_n - 1);
-			(*txq->elts)[elts_head] = buf;
+			(*txq->elts)[elts_head++ & elts_m] = buf;
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			for (n = 0; n * RTE_CACHE_LINE_SIZE < length; n++)
 				rte_prefetch2((void *)(addr +
@@ -1706,7 +1683,6 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			mpw_room -= (inl_pad + sizeof(*dseg));
 			inl_pad = 0;
 		}
-		elts_head = elts_head_next;
 #ifdef MLX5_PMD_SOFT_COUNTERS
 		/* Increment sent bytes counter. */
 		txq->stats.obytes += length;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 4195665f8..a21995030 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -242,8 +242,8 @@ struct hash_rxq {
 /* TX queue descriptor. */
 __extension__
 struct txq {
-	uint16_t elts_head; /* Current index in (*elts)[]. */
-	uint16_t elts_tail; /* First element awaiting completion. */
+	uint16_t elts_head; /* Current counter in (*elts)[]. */
+	uint16_t elts_tail; /* Counter of first element awaiting completion. */
 	uint16_t elts_comp; /* Counter since last completion request. */
 	uint16_t mpw_comp; /* WQ index since last completion request. */
 	uint16_t cq_ci; /* Consumer index for completion queue. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index bf72468d5..f0729a2a8 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -103,9 +103,10 @@ txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n)
 static void
 txq_free_elts(struct txq_ctrl *txq_ctrl)
 {
-	unsigned int elts_n = 1 << txq_ctrl->txq.elts_n;
-	unsigned int elts_head = txq_ctrl->txq.elts_head;
-	unsigned int elts_tail = txq_ctrl->txq.elts_tail;
+	const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	uint16_t elts_head = txq_ctrl->txq.elts_head;
+	uint16_t elts_tail = txq_ctrl->txq.elts_tail;
 	struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts;
 
 	DEBUG("%p: freeing WRs", (void *)txq_ctrl);
@@ -114,18 +115,17 @@ txq_free_elts(struct txq_ctrl *txq_ctrl)
 	txq_ctrl->txq.elts_comp = 0;
 
 	while (elts_tail != elts_head) {
-		struct rte_mbuf *elt = (*elts)[elts_tail];
+		struct rte_mbuf *elt = (*elts)[elts_tail & elts_m];
 
 		assert(elt != NULL);
 		rte_pktmbuf_free_seg(elt);
 #ifndef NDEBUG
 		/* Poisoning. */
-		memset(&(*elts)[elts_tail],
+		memset(&(*elts)[elts_tail & elts_m],
 		       0x77,
-		       sizeof((*elts)[elts_tail]));
+		       sizeof((*elts)[elts_tail & elts_m]));
 #endif
-		if (++elts_tail == elts_n)
-			elts_tail = 0;
+		++elts_tail;
 	}
 }
 
-- 
2.11.0

[dpdk-dev] [PATCH v4 2/5] net/mlx5: free buffers in bulk on Tx completion

[Yongseok Koh]

When processing Tx completion, it is more efficient to free buffers in bulk
using rte_mempool_put_bulk() if buffers are from a same mempool.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_rxtx.c | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 43db06ad8..66593679f 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -264,6 +264,9 @@ txq_complete(struct txq *txq)
 	uint16_t cq_ci = txq->cq_ci;
 	volatile struct mlx5_cqe *cqe = NULL;
 	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	struct rte_mempool *pool = NULL;
+	unsigned int blk_n = 0;
 
 	do {
 		volatile struct mlx5_cqe *tmp;
@@ -296,21 +299,33 @@ txq_complete(struct txq *txq)
 	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
 	/* Free buffers. */
 	while (elts_free != elts_tail) {
-		struct rte_mbuf *elt = (*txq->elts)[elts_free & elts_m];
-		struct rte_mbuf *elt_next =
-			(*txq->elts)[(elts_free + 1) & elts_m];
-
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (likely(m->pool == pool)) {
+				free[blk_n++] = m;
+			} else {
+				if (likely(pool != NULL))
+					rte_mempool_put_bulk(pool,
+							     (void *)free,
+							     blk_n);
+				free[0] = m;
+				pool = m->pool;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(pool, (void *)free, blk_n);
 #ifndef NDEBUG
-		/* Poisoning. */
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
 		memset(&(*txq->elts)[elts_free & elts_m],
 		       0x66,
 		       sizeof((*txq->elts)[elts_free & elts_m]));
-#endif
-		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
-		/* Only one segment needs to be freed. */
-		rte_pktmbuf_free_seg(elt);
 		++elts_free;
 	}
+#endif
 	txq->cq_ci = cq_ci;
 	txq->elts_tail = elts_tail;
 	/* Update the consumer index. */
-- 
2.11.0

[dpdk-dev] [PATCH v4 3/5] net/mlx5: use buffer address for LKEY search

[Yongseok Koh]

When searching LKEY, if search key is mempool pointer, the 2nd cacheline
has to be accessed and it even requires to check whether a buffer is
indirect per every search. Instead, using address for search key can reduce
cycles taken. And caching the last hit entry is beneficial as well.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_mr.c   | 17 ++++++++++++++---
 drivers/net/mlx5/mlx5_rxtx.c | 39 +++++++++++++++++++++------------------
 drivers/net/mlx5/mlx5_rxtx.h |  4 +++-
 drivers/net/mlx5/mlx5_txq.c  |  3 +--
 4 files changed, 39 insertions(+), 24 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 0a3638460..287335179 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -207,7 +207,8 @@ txq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx)
 			 sizeof(txq_ctrl->txq.mp2mr[0])));
 	}
 	/* Store the new entry. */
-	txq_ctrl->txq.mp2mr[idx].mp = mp;
+	txq_ctrl->txq.mp2mr[idx].start = (uintptr_t)mr->addr;
+	txq_ctrl->txq.mp2mr[idx].end = (uintptr_t)mr->addr + mr->length;
 	txq_ctrl->txq.mp2mr[idx].mr = mr;
 	txq_ctrl->txq.mp2mr[idx].lkey = htonl(mr->lkey);
 	DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
@@ -265,18 +266,28 @@ txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
 	struct txq_mp2mr_mbuf_check_data data = {
 		.ret = 0,
 	};
+	uintptr_t start;
+	uintptr_t end;
 	unsigned int i;
 
 	/* Register mempool only if the first element looks like a mbuf. */
 	if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
 			data.ret == -1)
 		return;
+	if (mlx5_check_mempool(mp, &start, &end) != 0) {
+		ERROR("mempool %p: not virtually contiguous",
+		      (void *)mp);
+		return;
+	}
 	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
-		if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) {
+		struct ibv_mr *mr = txq_ctrl->txq.mp2mr[i].mr;
+
+		if (unlikely(mr == NULL)) {
 			/* Unknown MP, add a new MR for it. */
 			break;
 		}
-		if (txq_ctrl->txq.mp2mr[i].mp == mp)
+		if (start >= (uintptr_t)mr->addr &&
+		    end <= (uintptr_t)mr->addr + mr->length)
 			return;
 	}
 	txq_mp2mr_reg(&txq_ctrl->txq, mp, i);
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 66593679f..688ee9028 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -77,7 +77,7 @@ static __rte_always_inline void
 txq_complete(struct txq *txq);
 
 static __rte_always_inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp);
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb);
 
 static __rte_always_inline void
 mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe);
@@ -352,7 +352,7 @@ txq_mb2mp(struct rte_mbuf *buf)
 }
 
 /**
- * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[].
+ * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
  * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
  * remove an entry first.
  *
@@ -365,27 +365,30 @@ txq_mb2mp(struct rte_mbuf *buf)
  *   mr->lkey on success, (uint32_t)-1 on failure.
  */
 static inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
+txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
 {
-	unsigned int i;
-	uint32_t lkey = (uint32_t)-1;
+	uint16_t i = txq->mr_cache_idx;
+	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
 
+	assert(i < RTE_DIM(txq->mp2mr));
+	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
+		return txq->mp2mr[i].lkey;
 	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
-		if (unlikely(txq->mp2mr[i].mp == NULL)) {
+		if (unlikely(txq->mp2mr[i].mr == NULL)) {
 			/* Unknown MP, add a new MR for it. */
 			break;
 		}
-		if (txq->mp2mr[i].mp == mp) {
+		if (txq->mp2mr[i].start <= addr &&
+		    txq->mp2mr[i].end >= addr) {
 			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
 			assert(htonl(txq->mp2mr[i].mr->lkey) ==
 			       txq->mp2mr[i].lkey);
-			lkey = txq->mp2mr[i].lkey;
-			break;
+			txq->mr_cache_idx = i;
+			return txq->mp2mr[i].lkey;
 		}
 	}
-	if (unlikely(lkey == (uint32_t)-1))
-		lkey = txq_mp2mr_reg(txq, mp, i);
-	return lkey;
+	txq->mr_cache_idx = 0;
+	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
 }
 
 /**
@@ -770,7 +773,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t){
 				htonl(length),
-				txq_mp2mr(txq, txq_mb2mp(buf)),
+				txq_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
@@ -809,7 +812,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		naddr = htonll(rte_pktmbuf_mtod(buf, uintptr_t));
 		*dseg = (rte_v128u32_t){
 			htonl(length),
-			txq_mp2mr(txq, txq_mb2mp(buf)),
+			txq_mb2mr(txq, buf),
 			naddr,
 			naddr >> 32,
 		};
@@ -1051,7 +1054,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
 				.byte_count = htonl(DATA_LEN(buf)),
-				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+				.lkey = txq_mb2mr(txq, buf),
 				.addr = htonll(addr),
 			};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1297,7 +1300,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+					.lkey = txq_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1604,7 +1607,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
+					.lkey = txq_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1687,7 +1690,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t) {
 				htonl(length),
-				txq_mp2mr(txq, txq_mb2mp(buf)),
+				txq_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index a21995030..d0f508e90 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -267,10 +267,12 @@ struct txq {
 	volatile uint32_t *cq_db; /* Completion queue doorbell. */
 	volatile void *bf_reg; /* Blueflame register. */
 	struct {
-		const struct rte_mempool *mp; /* Cached Memory Pool. */
+		uintptr_t start; /* Start address of MR */
+		uintptr_t end; /* End address of MR */
 		struct ibv_mr *mr; /* Memory Region (for mp). */
 		uint32_t lkey; /* htonl(mr->lkey) */
 	} mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MP to MR translation table. */
+	uint16_t mr_cache_idx; /* Index of last hit entry. */
 	struct rte_mbuf *(*elts)[]; /* TX elements. */
 	struct mlx5_txq_stats stats; /* TX queue counters. */
 } __rte_cache_aligned;
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index f0729a2a8..ac9dfc5f0 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -149,9 +149,8 @@ txq_cleanup(struct txq_ctrl *txq_ctrl)
 	if (txq_ctrl->cq != NULL)
 		claim_zero(ibv_destroy_cq(txq_ctrl->cq));
 	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
-		if (txq_ctrl->txq.mp2mr[i].mp == NULL)
+		if (txq_ctrl->txq.mp2mr[i].mr == NULL)
 			break;
-		assert(txq_ctrl->txq.mp2mr[i].mr != NULL);
 		claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr));
 	}
 	memset(txq_ctrl, 0, sizeof(*txq_ctrl));
-- 
2.11.0

[dpdk-dev] [PATCH v4 4/5] net/mlx5: select Rx/Tx callbacks when starting device

[Yongseok Koh]

The callbacks are global to a device but the seletion is made every queue
configuration, which is redundant.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/mlx5_ethdev.c  | 23 +++++++++++------------
 drivers/net/mlx5/mlx5_rxq.c     |  2 --
 drivers/net/mlx5/mlx5_trigger.c |  3 +++
 drivers/net/mlx5/mlx5_txq.c     |  2 --
 4 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 039335e2e..54c4c6cae 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -923,8 +923,6 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	struct priv *priv = dev->data->dev_private;
 	int ret = 0;
 	unsigned int i;
-	uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) =
-		mlx5_rx_burst;
 	unsigned int max_frame_len;
 	int rehash;
 	int restart = priv->started;
@@ -944,7 +942,7 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	/* Temporarily replace RX handler with a fake one, assuming it has not
 	 * been copied elsewhere. */
 	dev->rx_pkt_burst = removed_rx_burst;
-	/* Make sure everyone has left mlx5_rx_burst() and uses
+	/* Make sure everyone has left dev->rx_pkt_burst() and uses
 	 * removed_rx_burst() instead. */
 	rte_wmb();
 	usleep(1000);
@@ -1018,17 +1016,13 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 		/* Double fault, disable RX. */
 		break;
 	}
-	/*
-	 * Use a safe RX burst function in case of error, otherwise mimic
-	 * mlx5_dev_start().
-	 */
+	/* Mimic mlx5_dev_start(). */
 	if (ret) {
 		ERROR("unable to reconfigure RX queues, RX disabled");
-		rx_func = removed_rx_burst;
 	} else if (restart &&
-		 !rehash &&
-		 !priv_create_hash_rxqs(priv) &&
-		 !priv_rehash_flows(priv)) {
+		   !rehash &&
+		   !priv_create_hash_rxqs(priv) &&
+		   !priv_rehash_flows(priv)) {
 		if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_NONE)
 			priv_fdir_enable(priv);
 		priv_dev_interrupt_handler_install(priv, dev);
@@ -1036,7 +1030,12 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 	priv->mtu = mtu;
 	/* Burst functions can now be called again. */
 	rte_wmb();
-	dev->rx_pkt_burst = rx_func;
+	/*
+	 * Use a safe RX burst function in case of error, otherwise select RX
+	 * burst function again.
+	 */
+	if (!ret)
+		priv_select_rx_function(priv);
 out:
 	priv_unlock(priv);
 	assert(ret >= 0);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 22be999e8..98b20eb92 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1226,8 +1226,6 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		DEBUG("%p: adding RX queue %p to list",
 		      (void *)dev, (void *)rxq_ctrl);
 		(*priv->rxqs)[idx] = &rxq_ctrl->rxq;
-		/* Update receive callback. */
-		priv_select_rx_function(priv);
 	}
 	priv_unlock(priv);
 	return -ret;
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 40f23da93..595a9e064 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -72,6 +72,9 @@ mlx5_dev_start(struct rte_eth_dev *dev)
 		priv_unlock(priv);
 		return 0;
 	}
+	/* Update Rx/Tx callback. */
+	priv_select_tx_function(priv);
+	priv_select_rx_function(priv);
 	DEBUG("%p: allocating and configuring hash RX queues", (void *)dev);
 	err = priv_create_hash_rxqs(priv);
 	if (!err)
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index ac9dfc5f0..03e23f58d 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -496,8 +496,6 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		DEBUG("%p: adding TX queue %p to list",
 		      (void *)dev, (void *)txq_ctrl);
 		(*priv->txqs)[idx] = &txq_ctrl->txq;
-		/* Update send callback. */
-		priv_select_tx_function(priv);
 	}
 	priv_unlock(priv);
 	return -ret;
-- 
2.11.0

[dpdk-dev] [PATCH v4 5/5] net/mlx5: add vectorized Rx/Tx burst for x86

[Yongseok Koh]

To make vectorized burst routines enabled, it is required to run on x86_64
architecture. If all the conditions are met, the vectorized burst functions
are enabled automatically. The decision is made individually on RX and TX.
There's no PMD option to make a selection.

Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
---
 drivers/net/mlx5/Makefile            |    3 +
 drivers/net/mlx5/mlx5_defs.h         |   18 +
 drivers/net/mlx5/mlx5_ethdev.c       |   24 +-
 drivers/net/mlx5/mlx5_rxq.c          |   55 +-
 drivers/net/mlx5/mlx5_rxtx.c         |  387 +++-------
 drivers/net/mlx5/mlx5_rxtx.h         |  282 ++++++-
 drivers/net/mlx5/mlx5_rxtx_vec_sse.c | 1394 ++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_txq.c          |    2 +-
 8 files changed, 1868 insertions(+), 297 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rxtx_vec_sse.c

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 8b8f6eaac..8736de5d3 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -39,6 +39,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_txq.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx.c
+ifeq ($(CONFIG_RTE_ARCH_X86_64),y)
+SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rxtx_vec_sse.c
+endif
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_trigger.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mac.c
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 201bb3362..a76bc6f65 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -89,4 +89,22 @@
 /* Maximum Packet headers size (L2+L3+L4) for TSO. */
 #define MLX5_MAX_TSO_HEADER 128
 
+/* Default minimum number of Tx queues for vectorized Tx. */
+#define MLX5_VPMD_MIN_TXQS 4
+
+/* Threshold of buffer replenishment for vectorized Rx. */
+#define MLX5_VPMD_RXQ_RPLNSH_THRESH   64U
+
+/* Maximum size of burst for vectorized Rx. */
+#define MLX5_VPMD_RX_MAX_BURST        MLX5_VPMD_RXQ_RPLNSH_THRESH
+
+/*
+ * Maximum size of burst for vectorized Tx. This is related to the maximum size
+ * of Enhaned MPW (eMPW) WQE as vectorized Tx is supported with eMPW.
+ */
+#define MLX5_VPMD_TX_MAX_BURST        32U
+
+/* Number of packets vectorized Rx can simultaneously process in a loop. */
+#define MLX5_VPMD_DESCS_PER_LOOP      4
+
 #endif /* RTE_PMD_MLX5_DEFS_H_ */
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 54c4c6cae..b70b7b9a9 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -723,7 +723,8 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 
 	};
 
-	if (dev->rx_pkt_burst == mlx5_rx_burst)
+	if (dev->rx_pkt_burst == mlx5_rx_burst ||
+	    dev->rx_pkt_burst == mlx5_rx_burst_vec)
 		return ptypes;
 	return NULL;
 }
@@ -1585,9 +1586,16 @@ priv_select_tx_function(struct priv *priv)
 	priv->dev->tx_pkt_burst = mlx5_tx_burst;
 	/* Select appropriate TX function. */
 	if (priv->mps == MLX5_MPW_ENHANCED) {
-		priv->dev->tx_pkt_burst =
-			mlx5_tx_burst_empw;
-		DEBUG("selected Enhanced MPW TX function");
+		if (priv_check_vec_tx_support(priv) > 0) {
+			if (priv_check_raw_vec_tx_support(priv) > 0)
+				priv->dev->tx_pkt_burst = mlx5_tx_burst_raw_vec;
+			else
+				priv->dev->tx_pkt_burst = mlx5_tx_burst_vec;
+			DEBUG("selected Enhanced MPW TX vectorized function");
+		} else {
+			priv->dev->tx_pkt_burst = mlx5_tx_burst_empw;
+			DEBUG("selected Enhanced MPW TX function");
+		}
 	} else if (priv->mps && priv->txq_inline) {
 		priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline;
 		DEBUG("selected MPW inline TX function");
@@ -1606,5 +1614,11 @@ priv_select_tx_function(struct priv *priv)
 void
 priv_select_rx_function(struct priv *priv)
 {
-	priv->dev->rx_pkt_burst = mlx5_rx_burst;
+	if (priv_check_vec_rx_support(priv) > 0) {
+		priv_prep_vec_rx_function(priv);
+		priv->dev->rx_pkt_burst = mlx5_rx_burst_vec;
+		DEBUG("selected RX vectorized function");
+	} else {
+		priv->dev->rx_pkt_burst = mlx5_rx_burst;
+	}
 }
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 98b20eb92..34ec95bc5 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -634,6 +634,41 @@ priv_rehash_flows(struct priv *priv)
 }
 
 /**
+ * Unlike regular Rx function, vPMD Rx doesn't replace mbufs immediately when
+ * receiving packets. Instead it replaces later in bulk. In rxq->elts[], entries
+ * from rq_pi to rq_ci are owned by device but the rest is already delivered to
+ * application. In order not to reuse those mbufs by rxq_alloc_elts(), this
+ * function must be called to replace used mbufs.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+static int
+rxq_trim_elts(struct rxq *rxq)
+{
+	const uint16_t q_n = (1 << rxq->elts_n);
+	const uint16_t q_mask = q_n - 1;
+	uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi);
+	uint16_t i;
+
+	if (!rxq->trim_elts)
+		return 0;
+	for (i = 0; i < used; ++i) {
+		struct rte_mbuf *buf;
+		buf = rte_pktmbuf_alloc(rxq->mp);
+		if (!buf)
+			return ENOMEM;
+		(*rxq->elts)[(rxq->rq_ci + i) & q_mask] = buf;
+	}
+	rxq->rq_pi = rxq->rq_ci;
+	rxq->trim_elts = 0;
+	return 0;
+}
+
+/**
  * Allocate RX queue elements.
  *
  * @param rxq_ctrl
@@ -800,6 +835,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl)
 		return err;
 	}
 	/* Snatch mbufs from original queue. */
+	claim_zero(rxq_trim_elts(&rxq_ctrl->rxq));
 	claim_zero(rxq_alloc_elts(rxq_ctrl, elts_n, rxq_ctrl->rxq.elts));
 	for (i = 0; i != elts_n; ++i) {
 		struct rte_mbuf *buf = (*rxq_ctrl->rxq.elts)[i];
@@ -860,6 +896,7 @@ rxq_setup(struct rxq_ctrl *tmpl)
 	tmpl->rxq.cqe_n = log2above(cq_info.cqe_cnt);
 	tmpl->rxq.cq_ci = 0;
 	tmpl->rxq.rq_ci = 0;
+	tmpl->rxq.rq_pi = 0;
 	tmpl->rxq.cq_db = cq_info.dbrec;
 	tmpl->rxq.wqes =
 		(volatile struct mlx5_wqe_data_seg (*)[])
@@ -993,7 +1030,12 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
 	if (priv->cqe_comp) {
 		attr.cq.comp_mask |= IBV_EXP_CQ_INIT_ATTR_FLAGS;
 		attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE;
-		cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
+		/*
+		 * For vectorized Rx, it must not be doubled in order to
+		 * make cq_ci and rq_ci aligned.
+		 */
+		if (rxq_check_vec_support(&tmpl.rxq) < 0)
+			cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */
 	}
 	tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, tmpl.channel, 0,
 				    &attr.cq);
@@ -1103,7 +1145,9 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl,
 	if (rxq_ctrl->rxq.elts_n) {
 		assert(1 << rxq_ctrl->rxq.elts_n == desc);
 		assert(rxq_ctrl->rxq.elts != tmpl.rxq.elts);
-		ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts);
+		ret = rxq_trim_elts(&rxq_ctrl->rxq);
+		if (!ret)
+			ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts);
 	} else
 		ret = rxq_alloc_elts(&tmpl, desc, NULL);
 	if (ret) {
@@ -1165,6 +1209,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	struct priv *priv = dev->data->dev_private;
 	struct rxq *rxq = (*priv->rxqs)[idx];
 	struct rxq_ctrl *rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
+	const uint16_t desc_pad = MLX5_VPMD_DESCS_PER_LOOP; /* For vPMD. */
 	int ret;
 
 	if (mlx5_is_secondary())
@@ -1198,7 +1243,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		if (rxq_ctrl->rxq.elts_n != log2above(desc)) {
 			rxq_ctrl = rte_realloc(rxq_ctrl,
 					       sizeof(*rxq_ctrl) +
-					       desc * sizeof(struct rte_mbuf *),
+					       (desc + desc_pad) *
+						sizeof(struct rte_mbuf *),
 					       RTE_CACHE_LINE_SIZE);
 			if (!rxq_ctrl) {
 				ERROR("%p: unable to reallocate queue index %u",
@@ -1209,7 +1255,8 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		}
 	} else {
 		rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl) +
-					     desc * sizeof(struct rte_mbuf *),
+					     (desc + desc_pad) *
+					      sizeof(struct rte_mbuf *),
 					     0, socket);
 		if (rxq_ctrl == NULL) {
 			ERROR("%p: unable to allocate queue index %u",
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 688ee9028..ab6df19eb 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -69,19 +69,6 @@
 #include "mlx5_defs.h"
 #include "mlx5_prm.h"
 
-static __rte_always_inline int
-check_cqe(volatile struct mlx5_cqe *cqe,
-	  unsigned int cqes_n, const uint16_t ci);
-
-static __rte_always_inline void
-txq_complete(struct txq *txq);
-
-static __rte_always_inline uint32_t
-txq_mb2mr(struct txq *txq, struct rte_mbuf *mb);
-
-static __rte_always_inline void
-mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe);
-
 static __rte_always_inline uint32_t
 rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe);
 
@@ -92,101 +79,29 @@ mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe,
 static __rte_always_inline uint32_t
 rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe);
 
-#ifndef NDEBUG
-
-/**
- * Verify or set magic value in CQE.
- *
- * @param cqe
- *   Pointer to CQE.
- *
- * @return
- *   0 the first time.
- */
-static inline int
-check_cqe_seen(volatile struct mlx5_cqe *cqe)
-{
-	static const uint8_t magic[] = "seen";
-	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
-	int ret = 1;
-	unsigned int i;
-
-	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
-		if (!ret || (*buf)[i] != magic[i]) {
-			ret = 0;
-			(*buf)[i] = magic[i];
-		}
-	return ret;
-}
-
-#endif /* NDEBUG */
-
-/**
- * Check whether CQE is valid.
- *
- * @param cqe
- *   Pointer to CQE.
- * @param cqes_n
- *   Size of completion queue.
- * @param ci
- *   Consumer index.
- *
- * @return
- *   0 on success, 1 on failure.
- */
-static inline int
-check_cqe(volatile struct mlx5_cqe *cqe,
-	  unsigned int cqes_n, const uint16_t ci)
-{
-	uint16_t idx = ci & cqes_n;
-	uint8_t op_own = cqe->op_own;
-	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
-	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
-
-	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
-		return 1; /* No CQE. */
-#ifndef NDEBUG
-	if ((op_code == MLX5_CQE_RESP_ERR) ||
-	    (op_code == MLX5_CQE_REQ_ERR)) {
-		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
-		uint8_t syndrome = err_cqe->syndrome;
-
-		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
-		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
-			return 0;
-		if (!check_cqe_seen(cqe))
-			ERROR("unexpected CQE error %u (0x%02x)"
-			      " syndrome 0x%02x",
-			      op_code, op_code, syndrome);
-		return 1;
-	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
-		   (op_code != MLX5_CQE_REQ)) {
-		if (!check_cqe_seen(cqe))
-			ERROR("unexpected CQE opcode %u (0x%02x)",
-			      op_code, op_code);
-		return 1;
-	}
-#endif /* NDEBUG */
-	return 0;
-}
-
-/**
- * Return the address of the WQE.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param  wqe_ci
- *   WQE consumer index.
- *
- * @return
- *   WQE address.
+/*
+ * The index to the array should have:
+ * bit[1:0] = l3_hdr_type, bit[2] = tunneled, bit[3] = outer_l3_type
  */
-static inline uintptr_t *
-tx_mlx5_wqe(struct txq *txq, uint16_t ci)
-{
-	ci &= ((1 << txq->wqe_n) - 1);
-	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
-}
+const uint32_t mlx5_ptype_table[] = {
+	RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b0001 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b0010 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b0101 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b0110 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,               /* b1001 */
+	RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,               /* b1010 */
+	RTE_PTYPE_UNKNOWN, RTE_PTYPE_UNKNOWN,
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, /* b1101 */
+	RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, /* b1110 */
+	RTE_PTYPE_ALL_MASK			     /* b1111 */
+};
 
 /**
  * Return the size of tailroom of WQ.
@@ -245,174 +160,6 @@ mlx5_copy_to_wq(void *dst, const void *src, size_t n,
 }
 
 /**
- * Manage TX completions.
- *
- * When sending a burst, mlx5_tx_burst() posts several WRs.
- *
- * @param txq
- *   Pointer to TX queue structure.
- */
-static inline void
-txq_complete(struct txq *txq)
-{
-	const uint16_t elts_n = 1 << txq->elts_n;
-	const uint16_t elts_m = elts_n - 1;
-	const unsigned int cqe_n = 1 << txq->cqe_n;
-	const unsigned int cqe_cnt = cqe_n - 1;
-	uint16_t elts_free = txq->elts_tail;
-	uint16_t elts_tail;
-	uint16_t cq_ci = txq->cq_ci;
-	volatile struct mlx5_cqe *cqe = NULL;
-	volatile struct mlx5_wqe_ctrl *ctrl;
-	struct rte_mbuf *m, *free[elts_n];
-	struct rte_mempool *pool = NULL;
-	unsigned int blk_n = 0;
-
-	do {
-		volatile struct mlx5_cqe *tmp;
-
-		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
-		if (check_cqe(tmp, cqe_n, cq_ci))
-			break;
-		cqe = tmp;
-#ifndef NDEBUG
-		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
-			if (!check_cqe_seen(cqe))
-				ERROR("unexpected compressed CQE, TX stopped");
-			return;
-		}
-		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
-		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
-			if (!check_cqe_seen(cqe))
-				ERROR("unexpected error CQE, TX stopped");
-			return;
-		}
-#endif /* NDEBUG */
-		++cq_ci;
-	} while (1);
-	if (unlikely(cqe == NULL))
-		return;
-	txq->wqe_pi = ntohs(cqe->wqe_counter);
-	ctrl = (volatile struct mlx5_wqe_ctrl *)
-		tx_mlx5_wqe(txq, txq->wqe_pi);
-	elts_tail = ctrl->ctrl3;
-	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
-	/* Free buffers. */
-	while (elts_free != elts_tail) {
-		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
-		if (likely(m != NULL)) {
-			if (likely(m->pool == pool)) {
-				free[blk_n++] = m;
-			} else {
-				if (likely(pool != NULL))
-					rte_mempool_put_bulk(pool,
-							     (void *)free,
-							     blk_n);
-				free[0] = m;
-				pool = m->pool;
-				blk_n = 1;
-			}
-		}
-	}
-	if (blk_n)
-		rte_mempool_put_bulk(pool, (void *)free, blk_n);
-#ifndef NDEBUG
-	elts_free = txq->elts_tail;
-	/* Poisoning. */
-	while (elts_free != elts_tail) {
-		memset(&(*txq->elts)[elts_free & elts_m],
-		       0x66,
-		       sizeof((*txq->elts)[elts_free & elts_m]));
-		++elts_free;
-	}
-#endif
-	txq->cq_ci = cq_ci;
-	txq->elts_tail = elts_tail;
-	/* Update the consumer index. */
-	rte_wmb();
-	*txq->cq_db = htonl(cq_ci);
-}
-
-/**
- * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
- * the cloned mbuf is allocated is returned instead.
- *
- * @param buf
- *   Pointer to mbuf.
- *
- * @return
- *   Memory pool where data is located for given mbuf.
- */
-static struct rte_mempool *
-txq_mb2mp(struct rte_mbuf *buf)
-{
-	if (unlikely(RTE_MBUF_INDIRECT(buf)))
-		return rte_mbuf_from_indirect(buf)->pool;
-	return buf->pool;
-}
-
-/**
- * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
- * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
- * remove an entry first.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param[in] mp
- *   Memory Pool for which a Memory Region lkey must be returned.
- *
- * @return
- *   mr->lkey on success, (uint32_t)-1 on failure.
- */
-static inline uint32_t
-txq_mb2mr(struct txq *txq, struct rte_mbuf *mb)
-{
-	uint16_t i = txq->mr_cache_idx;
-	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
-
-	assert(i < RTE_DIM(txq->mp2mr));
-	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
-		return txq->mp2mr[i].lkey;
-	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
-		if (unlikely(txq->mp2mr[i].mr == NULL)) {
-			/* Unknown MP, add a new MR for it. */
-			break;
-		}
-		if (txq->mp2mr[i].start <= addr &&
-		    txq->mp2mr[i].end >= addr) {
-			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
-			assert(htonl(txq->mp2mr[i].mr->lkey) ==
-			       txq->mp2mr[i].lkey);
-			txq->mr_cache_idx = i;
-			return txq->mp2mr[i].lkey;
-		}
-	}
-	txq->mr_cache_idx = 0;
-	return txq_mp2mr_reg(txq, txq_mb2mp(mb), i);
-}
-
-/**
- * Ring TX queue doorbell.
- *
- * @param txq
- *   Pointer to TX queue structure.
- * @param wqe
- *   Pointer to the last WQE posted in the NIC.
- */
-static inline void
-mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
-{
-	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
-	volatile uint64_t *src = ((volatile uint64_t *)wqe);
-
-	rte_wmb();
-	*txq->qp_db = htonl(txq->wqe_ci);
-	/* Ensure ordering between DB record and BF copy. */
-	rte_wmb();
-	*dst = *src;
-}
-
-/**
  * DPDK callback to check the status of a tx descriptor.
  *
  * @param tx_queue
@@ -429,7 +176,7 @@ mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset)
 	struct txq *txq = tx_queue;
 	uint16_t used;
 
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	used = txq->elts_head - txq->elts_tail;
 	if (offset < used)
 		return RTE_ETH_TX_DESC_FULL;
@@ -525,7 +272,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	/* Prefetch first packet cacheline. */
 	rte_prefetch0(*pkts);
 	/* Start processing. */
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
@@ -773,7 +520,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t){
 				htonl(length),
-				txq_mb2mr(txq, buf),
+				mlx5_tx_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
@@ -812,7 +559,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		naddr = htonll(rte_pktmbuf_mtod(buf, uintptr_t));
 		*dseg = (rte_v128u32_t){
 			htonl(length),
-			txq_mb2mr(txq, buf),
+			mlx5_tx_mb2mr(txq, buf),
 			naddr,
 			naddr >> 32,
 		};
@@ -991,7 +738,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
 	if (unlikely(!max_wqe))
@@ -1054,7 +801,7 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			addr = rte_pktmbuf_mtod(buf, uintptr_t);
 			*dseg = (struct mlx5_wqe_data_seg){
 				.byte_count = htonl(DATA_LEN(buf)),
-				.lkey = txq_mb2mr(txq, buf),
+				.lkey = mlx5_tx_mb2mr(txq, buf),
 				.addr = htonll(addr),
 			};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1214,7 +961,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci));
 	rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
 	/* Start processing. */
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	do {
 		struct rte_mbuf *buf = *(pkts++);
@@ -1300,7 +1047,7 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mb2mr(txq, buf),
+					.lkey = mlx5_tx_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1495,7 +1242,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	if (unlikely(!pkts_n))
 		return 0;
 	/* Start processing. */
-	txq_complete(txq);
+	mlx5_tx_complete(txq);
 	max_elts = (elts_n - (elts_head - txq->elts_tail));
 	/* A CQE slot must always be available. */
 	assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
@@ -1607,7 +1354,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				addr = rte_pktmbuf_mtod(buf, uintptr_t);
 				*dseg = (struct mlx5_wqe_data_seg){
 					.byte_count = htonl(DATA_LEN(buf)),
-					.lkey = txq_mb2mr(txq, buf),
+					.lkey = mlx5_tx_mb2mr(txq, buf),
 					.addr = htonll(addr),
 				};
 #if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
@@ -1690,7 +1437,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			naddr = htonll(addr);
 			*dseg = (rte_v128u32_t) {
 				htonl(length),
-				txq_mb2mr(txq, buf),
+				mlx5_tx_mb2mr(txq, buf),
 				naddr,
 				naddr >> 32,
 			};
@@ -2138,3 +1885,71 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	(void)pkts_n;
 	return 0;
 }
+
+/*
+ * Vectorized Rx/Tx routines are not compiled in when required vector
+ * instructions are not supported on a target architecture. The following null
+ * stubs are needed for linkage when those are not included outside of this file
+ * (e.g.  mlx5_rxtx_vec_sse.c for x86).
+ */
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_txq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_txq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+uint16_t __attribute__((weak))
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	(void)dpdk_rxq;
+	(void)pkts;
+	(void)pkts_n;
+	return 0;
+}
+
+int __attribute__((weak))
+priv_check_raw_vec_tx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_vec_tx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+rxq_check_vec_support(struct rxq *rxq)
+{
+	(void)rxq;
+	return -ENOTSUP;
+}
+
+int __attribute__((weak))
+priv_check_vec_rx_support(struct priv *priv)
+{
+	(void)priv;
+	return -ENOTSUP;
+}
+
+void __attribute__((weak))
+priv_prep_vec_rx_function(struct priv *priv)
+{
+	(void)priv;
+}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index d0f508e90..534aaeb46 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -115,10 +115,13 @@ struct rxq {
 	unsigned int port_id:8;
 	unsigned int rss_hash:1; /* RSS hash result is enabled. */
 	unsigned int mark:1; /* Marked flow available on the queue. */
-	unsigned int :8; /* Remaining bits. */
+	unsigned int pending_err:1; /* CQE error needs to be handled. */
+	unsigned int trim_elts:1; /* Whether elts needs clean-up. */
+	unsigned int :6; /* Remaining bits. */
 	volatile uint32_t *rq_db;
 	volatile uint32_t *cq_db;
 	uint16_t rq_ci;
+	uint16_t rq_pi;
 	uint16_t cq_ci;
 	volatile struct mlx5_wqe_data_seg(*wqes)[];
 	volatile struct mlx5_cqe(*cqes)[];
@@ -126,6 +129,8 @@ struct rxq {
 	struct rte_mbuf *(*elts)[];
 	struct rte_mempool *mp;
 	struct mlx5_rxq_stats stats;
+	uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */
+	struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
 } __rte_cache_aligned;
 
 /* RX queue control descriptor. */
@@ -261,6 +266,7 @@ struct txq {
 	uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */
 	uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
 	uint32_t qp_num_8s; /* QP number shifted by 8. */
+	uint32_t flags; /* Flags for Tx Queue. */
 	volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
 	volatile void *wqes; /* Work queue (use volatile to write into). */
 	volatile uint32_t *qp_db; /* Work queue doorbell. */
@@ -328,6 +334,8 @@ uint16_t mlx5_tx_burst_secondary_setup(void *, struct rte_mbuf **, uint16_t);
 
 /* mlx5_rxtx.c */
 
+extern const uint32_t mlx5_ptype_table[];
+
 uint16_t mlx5_tx_burst(void *, struct rte_mbuf **, uint16_t);
 uint16_t mlx5_tx_burst_mpw(void *, struct rte_mbuf **, uint16_t);
 uint16_t mlx5_tx_burst_mpw_inline(void *, struct rte_mbuf **, uint16_t);
@@ -338,10 +346,282 @@ uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t);
 int mlx5_rx_descriptor_status(void *, uint16_t);
 int mlx5_tx_descriptor_status(void *, uint16_t);
 
+/* Vectorized version of mlx5_rxtx.c */
+int priv_check_raw_vec_tx_support(struct priv *);
+int priv_check_vec_tx_support(struct priv *);
+int rxq_check_vec_support(struct rxq *);
+int priv_check_vec_rx_support(struct priv *);
+void priv_prep_vec_rx_function(struct priv *);
+uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t);
+uint16_t mlx5_tx_burst_vec(void *, struct rte_mbuf **, uint16_t);
+uint16_t mlx5_rx_burst_vec(void *, struct rte_mbuf **, uint16_t);
+
 /* mlx5_mr.c */
 
 struct ibv_mr *mlx5_mp2mr(struct ibv_pd *, struct rte_mempool *);
 void txq_mp2mr_iter(struct rte_mempool *, void *);
 uint32_t txq_mp2mr_reg(struct txq *, struct rte_mempool *, unsigned int);
 
+#ifndef NDEBUG
+/**
+ * Verify or set magic value in CQE.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ *
+ * @return
+ *   0 the first time.
+ */
+static inline int
+check_cqe_seen(volatile struct mlx5_cqe *cqe)
+{
+	static const uint8_t magic[] = "seen";
+	volatile uint8_t (*buf)[sizeof(cqe->rsvd0)] = &cqe->rsvd0;
+	int ret = 1;
+	unsigned int i;
+
+	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
+		if (!ret || (*buf)[i] != magic[i]) {
+			ret = 0;
+			(*buf)[i] = magic[i];
+		}
+	return ret;
+}
+#endif /* NDEBUG */
+
+/**
+ * Check whether CQE is valid.
+ *
+ * @param cqe
+ *   Pointer to CQE.
+ * @param cqes_n
+ *   Size of completion queue.
+ * @param ci
+ *   Consumer index.
+ *
+ * @return
+ *   0 on success, 1 on failure.
+ */
+static __rte_always_inline int
+check_cqe(volatile struct mlx5_cqe *cqe,
+	  unsigned int cqes_n, const uint16_t ci)
+{
+	uint16_t idx = ci & cqes_n;
+	uint8_t op_own = cqe->op_own;
+	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
+	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
+
+	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
+		return 1; /* No CQE. */
+#ifndef NDEBUG
+	if ((op_code == MLX5_CQE_RESP_ERR) ||
+	    (op_code == MLX5_CQE_REQ_ERR)) {
+		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
+		uint8_t syndrome = err_cqe->syndrome;
+
+		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
+		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
+			return 0;
+		if (!check_cqe_seen(cqe))
+			ERROR("unexpected CQE error %u (0x%02x)"
+			      " syndrome 0x%02x",
+			      op_code, op_code, syndrome);
+		return 1;
+	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
+		   (op_code != MLX5_CQE_REQ)) {
+		if (!check_cqe_seen(cqe))
+			ERROR("unexpected CQE opcode %u (0x%02x)",
+			      op_code, op_code);
+		return 1;
+	}
+#endif /* NDEBUG */
+	return 0;
+}
+
+/**
+ * Return the address of the WQE.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param  wqe_ci
+ *   WQE consumer index.
+ *
+ * @return
+ *   WQE address.
+ */
+static inline uintptr_t *
+tx_mlx5_wqe(struct txq *txq, uint16_t ci)
+{
+	ci &= ((1 << txq->wqe_n) - 1);
+	return (uintptr_t *)((uintptr_t)txq->wqes + ci * MLX5_WQE_SIZE);
+}
+
+/**
+ * Manage TX completions.
+ *
+ * When sending a burst, mlx5_tx_burst() posts several WRs.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ */
+static __rte_always_inline void
+mlx5_tx_complete(struct txq *txq)
+{
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int cqe_n = 1 << txq->cqe_n;
+	const unsigned int cqe_cnt = cqe_n - 1;
+	uint16_t elts_free = txq->elts_tail;
+	uint16_t elts_tail;
+	uint16_t cq_ci = txq->cq_ci;
+	volatile struct mlx5_cqe *cqe = NULL;
+	volatile struct mlx5_wqe_ctrl *ctrl;
+	struct rte_mbuf *m, *free[elts_n];
+	struct rte_mempool *pool = NULL;
+	unsigned int blk_n = 0;
+
+	do {
+		volatile struct mlx5_cqe *tmp;
+
+		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
+		if (check_cqe(tmp, cqe_n, cq_ci))
+			break;
+		cqe = tmp;
+#ifndef NDEBUG
+		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
+			if (!check_cqe_seen(cqe))
+				ERROR("unexpected compressed CQE, TX stopped");
+			return;
+		}
+		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
+		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
+			if (!check_cqe_seen(cqe))
+				ERROR("unexpected error CQE, TX stopped");
+			return;
+		}
+#endif /* NDEBUG */
+		++cq_ci;
+	} while (1);
+	if (unlikely(cqe == NULL))
+		return;
+	txq->wqe_pi = ntohs(cqe->wqe_counter);
+	ctrl = (volatile struct mlx5_wqe_ctrl *)
+		tx_mlx5_wqe(txq, txq->wqe_pi);
+	elts_tail = ctrl->ctrl3;
+	assert((elts_tail & elts_m) < (1 << txq->wqe_n));
+	/* Free buffers. */
+	while (elts_free != elts_tail) {
+		m = rte_pktmbuf_prefree_seg((*txq->elts)[elts_free++ & elts_m]);
+		if (likely(m != NULL)) {
+			if (likely(m->pool == pool)) {
+				free[blk_n++] = m;
+			} else {
+				if (likely(pool != NULL))
+					rte_mempool_put_bulk(pool,
+							     (void *)free,
+							     blk_n);
+				free[0] = m;
+				pool = m->pool;
+				blk_n = 1;
+			}
+		}
+	}
+	if (blk_n)
+		rte_mempool_put_bulk(pool, (void *)free, blk_n);
+#ifndef NDEBUG
+	elts_free = txq->elts_tail;
+	/* Poisoning. */
+	while (elts_free != elts_tail) {
+		memset(&(*txq->elts)[elts_free & elts_m],
+		       0x66,
+		       sizeof((*txq->elts)[elts_free & elts_m]));
+		++elts_free;
+	}
+#endif
+	txq->cq_ci = cq_ci;
+	txq->elts_tail = elts_tail;
+	/* Update the consumer index. */
+	rte_wmb();
+	*txq->cq_db = htonl(cq_ci);
+}
+
+/**
+ * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
+ * the cloned mbuf is allocated is returned instead.
+ *
+ * @param buf
+ *   Pointer to mbuf.
+ *
+ * @return
+ *   Memory pool where data is located for given mbuf.
+ */
+static struct rte_mempool *
+mlx5_tx_mb2mp(struct rte_mbuf *buf)
+{
+	if (unlikely(RTE_MBUF_INDIRECT(buf)))
+		return rte_mbuf_from_indirect(buf)->pool;
+	return buf->pool;
+}
+
+/**
+ * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
+ * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
+ * remove an entry first.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param[in] mp
+ *   Memory Pool for which a Memory Region lkey must be returned.
+ *
+ * @return
+ *   mr->lkey on success, (uint32_t)-1 on failure.
+ */
+static __rte_always_inline uint32_t
+mlx5_tx_mb2mr(struct txq *txq, struct rte_mbuf *mb)
+{
+	uint16_t i = txq->mr_cache_idx;
+	uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
+
+	assert(i < RTE_DIM(txq->mp2mr));
+	if (likely(txq->mp2mr[i].start <= addr && txq->mp2mr[i].end >= addr))
+		return txq->mp2mr[i].lkey;
+	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
+		if (unlikely(txq->mp2mr[i].mr == NULL)) {
+			/* Unknown MP, add a new MR for it. */
+			break;
+		}
+		if (txq->mp2mr[i].start <= addr &&
+		    txq->mp2mr[i].end >= addr) {
+			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
+			assert(htonl(txq->mp2mr[i].mr->lkey) ==
+			       txq->mp2mr[i].lkey);
+			txq->mr_cache_idx = i;
+			return txq->mp2mr[i].lkey;
+		}
+	}
+	txq->mr_cache_idx = 0;
+	return txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i);
+}
+
+/**
+ * Ring TX queue doorbell.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param wqe
+ *   Pointer to the last WQE posted in the NIC.
+ */
+static __rte_always_inline void
+mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
+{
+	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
+	volatile uint64_t *src = ((volatile uint64_t *)wqe);
+
+	rte_wmb();
+	*txq->qp_db = htonl(txq->wqe_ci);
+	/* Ensure ordering between DB record and BF copy. */
+	rte_wmb();
+	*dst = *src;
+}
+
 #endif /* RTE_PMD_MLX5_RXTX_H_ */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.c b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
new file mode 100644
index 000000000..95e96baa2
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.c
@@ -0,0 +1,1394 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <smmintrin.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5_hw.h>
+#include <infiniband/arch.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+/* DPDK headers don't like -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include "mlx5.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5_prm.h"
+
+#ifndef __INTEL_COMPILER
+#pragma GCC diagnostic ignored "-Wcast-qual"
+#endif
+
+/**
+ * Fill in buffer descriptors in a multi-packet send descriptor.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param dseg
+ *   Pointer to buffer descriptor to be writen.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param n
+ *   Number of packets to be filled.
+ */
+static inline void
+txq_wr_dseg_v(struct txq *txq, __m128i *dseg,
+	      struct rte_mbuf **pkts, unsigned int n)
+{
+	unsigned int pos;
+	uintptr_t addr;
+	const __m128i shuf_mask_dseg =
+		_mm_set_epi8(8,  9, 10, 11, /* addr, bswap64 */
+			    12, 13, 14, 15,
+			     7,  6,  5,  4, /* lkey */
+			     0,  1,  2,  3  /* length, bswap32 */);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t tx_byte = 0;
+#endif
+
+	for (pos = 0; pos < n; ++pos, ++dseg) {
+		__m128i desc;
+		struct rte_mbuf *pkt = pkts[pos];
+
+		addr = rte_pktmbuf_mtod(pkt, uintptr_t);
+		desc = _mm_set_epi32(addr >> 32,
+				     addr,
+				     mlx5_tx_mb2mr(txq, pkt),
+				     DATA_LEN(pkt));
+		desc = _mm_shuffle_epi8(desc, shuf_mask_dseg);
+		_mm_store_si128(dseg, desc);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		tx_byte += DATA_LEN(pkt);
+#endif
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.obytes += tx_byte;
+#endif
+}
+
+/**
+ * Count the number of continuous single segment packets. The first packet must
+ * be a single segment packet.
+ *
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ *
+ * @return
+ *   Number of continuous single segment packets.
+ */
+static inline unsigned int
+txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	unsigned int pos;
+
+	if (!pkts_n)
+		return 0;
+	assert(NB_SEGS(pkts[0]) == 1);
+	/* Count the number of continuous single segment packets. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if (NB_SEGS(pkts[pos]) > 1)
+			break;
+	return pos;
+}
+
+/**
+ * Count the number of packets having same ol_flags and calculate cs_flags.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets.
+ * @param pkts_n
+ *   Number of packets.
+ * @param cs_flags
+ *   Pointer of flags to be returned.
+ *
+ * @return
+ *   Number of packets having same ol_flags.
+ */
+static inline unsigned int
+txq_calc_offload(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+		 uint8_t *cs_flags)
+{
+	unsigned int pos;
+	const uint64_t ol_mask =
+		PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM |
+		PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE |
+		PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM;
+
+	if (!pkts_n)
+		return 0;
+	/* Count the number of packets having same ol_flags. */
+	for (pos = 1; pos < pkts_n; ++pos)
+		if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask)
+			break;
+	/* Should open another MPW session for the rest. */
+	if (pkts[0]->ol_flags &
+	    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+		const uint64_t is_tunneled =
+			pkts[0]->ol_flags &
+			(PKT_TX_TUNNEL_GRE |
+			 PKT_TX_TUNNEL_VXLAN);
+
+		if (is_tunneled && txq->tunnel_en) {
+			*cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
+				    MLX5_ETH_WQE_L4_INNER_CSUM;
+			if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+				*cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+		} else {
+			*cs_flags = MLX5_ETH_WQE_L3_CSUM |
+				    MLX5_ETH_WQE_L4_CSUM;
+		}
+	}
+	return pos;
+}
+
+/**
+ * Send multi-segmented packets until it encounters a single segment packet in
+ * the pkts list.
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static uint16_t
+txq_scatter_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n;
+	volatile struct mlx5_wqe *wqe = NULL;
+
+	assert(elts_n > pkts_n);
+	mlx5_tx_complete(txq);
+	if (unlikely(!pkts_n))
+		return 0;
+	for (n = 0; n < pkts_n; ++n) {
+		struct rte_mbuf *buf = pkts[n];
+		unsigned int segs_n = buf->nb_segs;
+		unsigned int ds = nb_dword_in_hdr;
+		unsigned int len = PKT_LEN(buf);
+		uint16_t wqe_ci = txq->wqe_ci;
+		const __m128i shuf_mask_ctrl =
+			_mm_set_epi8(15, 14, 13, 12,
+				      8,  9, 10, 11, /* bswap32 */
+				      4,  5,  6,  7, /* bswap32 */
+				      0,  1,  2,  3  /* bswap32 */);
+		uint8_t cs_flags = 0;
+		uint16_t max_elts;
+		uint16_t max_wqe;
+		__m128i *t_wqe, *dseg;
+		__m128i ctrl;
+
+		assert(segs_n);
+		max_elts = elts_n - (elts_head - txq->elts_tail);
+		max_wqe = wq_n - (txq->wqe_ci - txq->wqe_pi);
+		/*
+		 * A MPW session consumes 2 WQEs at most to
+		 * include MLX5_MPW_DSEG_MAX pointers.
+		 */
+		if (segs_n == 1 ||
+		    max_elts < segs_n || max_wqe < 2)
+			break;
+		wqe = &((volatile struct mlx5_wqe64 *)
+			 txq->wqes)[wqe_ci & wq_mask].hdr;
+		if (buf->ol_flags &
+		     (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
+			const uint64_t is_tunneled = buf->ol_flags &
+						      (PKT_TX_TUNNEL_GRE |
+						       PKT_TX_TUNNEL_VXLAN);
+
+			if (is_tunneled && txq->tunnel_en) {
+				cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM |
+					   MLX5_ETH_WQE_L4_INNER_CSUM;
+				if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+					cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+			} else {
+				cs_flags = MLX5_ETH_WQE_L3_CSUM |
+					   MLX5_ETH_WQE_L4_CSUM;
+			}
+		}
+		/* Title WQEBB pointer. */
+		t_wqe = (__m128i *)wqe;
+		dseg = (__m128i *)(wqe + 1);
+		do {
+			if (!(ds++ % nb_dword_per_wqebb)) {
+				dseg = (__m128i *)
+					&((volatile struct mlx5_wqe64 *)
+					   txq->wqes)[++wqe_ci & wq_mask];
+			}
+			txq_wr_dseg_v(txq, dseg++, &buf, 1);
+			(*txq->elts)[elts_head++ & elts_m] = buf;
+			buf = buf->next;
+		} while (--segs_n);
+		if (ds % nb_dword_per_wqebb)
+			++wqe_ci;
+		/* Fill CTRL in the header. */
+		ctrl = _mm_set_epi32(0, 0, txq->qp_num_8s | ds,
+				     MLX5_OPC_MOD_MPW << 24 |
+				     txq->wqe_ci << 8 | MLX5_OPCODE_TSO);
+		ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+		_mm_store_si128(t_wqe, ctrl);
+		/* Fill ESEG in the header. */
+		_mm_store_si128(t_wqe + 1,
+				_mm_set_epi16(0, 0, 0, 0,
+					      htons(len), cs_flags,
+					      0, 0));
+		txq->wqe_ci = wqe_ci;
+	}
+	if (!n)
+		return 0;
+	txq->elts_comp += (uint16_t)(elts_head - txq->elts_head);
+	txq->elts_head = elts_head;
+	if (txq->elts_comp >= MLX5_TX_COMP_THRESH) {
+		wqe->ctrl[2] = htonl(8);
+		wqe->ctrl[3] = txq->elts_head;
+		txq->elts_comp = 0;
+		++txq->cq_pi;
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += n;
+#endif
+	mlx5_tx_dbrec(txq, wqe);
+	return n;
+}
+
+/**
+ * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
+ * it returns to make it processed by txq_scatter_v(). All the packets in
+ * the pkts list should be single segment packets having same offload flags.
+ * This must be checked by txq_check_multiseg() and txq_calc_offload().
+ *
+ * @param txq
+ *   Pointer to TX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be sent.
+ * @param pkts_n
+ *   Number of packets to be sent (<= MLX5_VPMD_TX_MAX_BURST).
+ * @param cs_flags
+ *   Checksum offload flags to be written in the descriptor.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+static inline uint16_t
+txq_burst_v(struct txq *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
+	    uint8_t cs_flags)
+{
+	struct rte_mbuf **elts;
+	uint16_t elts_head = txq->elts_head;
+	const uint16_t elts_n = 1 << txq->elts_n;
+	const uint16_t elts_m = elts_n - 1;
+	const unsigned int nb_dword_per_wqebb =
+		MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE;
+	const unsigned int nb_dword_in_hdr =
+		sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
+	unsigned int n = 0;
+	unsigned int pos;
+	uint16_t max_elts;
+	uint16_t max_wqe;
+	uint32_t comp_req = 0;
+	const uint16_t wq_n = 1 << txq->wqe_n;
+	const uint16_t wq_mask = wq_n - 1;
+	uint16_t wq_idx = txq->wqe_ci & wq_mask;
+	volatile struct mlx5_wqe64 *wq =
+		&((volatile struct mlx5_wqe64 *)txq->wqes)[wq_idx];
+	volatile struct mlx5_wqe *wqe = (volatile struct mlx5_wqe *)wq;
+	const __m128i shuf_mask_ctrl =
+		_mm_set_epi8(15, 14, 13, 12,
+			      8,  9, 10, 11, /* bswap32 */
+			      4,  5,  6,  7, /* bswap32 */
+			      0,  1,  2,  3  /* bswap32 */);
+	__m128i *t_wqe, *dseg;
+	__m128i ctrl;
+
+	/* Make sure all packets can fit into a single WQE. */
+	assert(elts_n > pkts_n);
+	mlx5_tx_complete(txq);
+	max_elts = (elts_n - (elts_head - txq->elts_tail));
+	max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
+	pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
+	if (unlikely(!pkts_n))
+		return 0;
+	elts = &(*txq->elts)[elts_head & elts_m];
+	/* Loop for available tailroom first. */
+	n = RTE_MIN(elts_n - (elts_head & elts_m), pkts_n);
+	for (pos = 0; pos < (n & -2); pos += 2)
+		_mm_storeu_si128((__m128i *)&elts[pos],
+				 _mm_loadu_si128((__m128i *)&pkts[pos]));
+	if (n & 1)
+		elts[pos] = pkts[pos];
+	/* Check if it crosses the end of the queue. */
+	if (unlikely(n < pkts_n)) {
+		elts = &(*txq->elts)[0];
+		for (pos = 0; pos < pkts_n - n; ++pos)
+			elts[pos] = pkts[n + pos];
+	}
+	txq->elts_head += pkts_n;
+	/* Save title WQEBB pointer. */
+	t_wqe = (__m128i *)wqe;
+	dseg = (__m128i *)(wqe + 1);
+	/* Calculate the number of entries to the end. */
+	n = RTE_MIN(
+		(wq_n - wq_idx) * nb_dword_per_wqebb - nb_dword_in_hdr,
+		pkts_n);
+	/* Fill DSEGs. */
+	txq_wr_dseg_v(txq, dseg, pkts, n);
+	/* Check if it crosses the end of the queue. */
+	if (n < pkts_n) {
+		dseg = (__m128i *)txq->wqes;
+		txq_wr_dseg_v(txq, dseg, &pkts[n], pkts_n - n);
+	}
+	if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) {
+		txq->elts_comp += pkts_n;
+	} else {
+		/* Request a completion. */
+		txq->elts_comp = 0;
+		++txq->cq_pi;
+		comp_req = 8;
+	}
+	/* Fill CTRL in the header. */
+	ctrl = _mm_set_epi32(txq->elts_head, comp_req,
+			     txq->qp_num_8s | (pkts_n + 2),
+			     MLX5_OPC_MOD_ENHANCED_MPSW << 24 |
+				txq->wqe_ci << 8 | MLX5_OPCODE_ENHANCED_MPSW);
+	ctrl = _mm_shuffle_epi8(ctrl, shuf_mask_ctrl);
+	_mm_store_si128(t_wqe, ctrl);
+	/* Fill ESEG in the header. */
+	_mm_store_si128(t_wqe + 1,
+			_mm_set_epi8(0, 0, 0, 0,
+				     0, 0, 0, 0,
+				     0, 0, 0, cs_flags,
+				     0, 0, 0, 0));
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	txq->stats.opackets += pkts_n;
+#endif
+	txq->wqe_ci += (nb_dword_in_hdr + pkts_n + (nb_dword_per_wqebb - 1)) /
+		       nb_dword_per_wqebb;
+	/* Ring QP doorbell. */
+	mlx5_tx_dbrec(txq, wqe);
+	return pkts_n;
+}
+
+/**
+ * DPDK callback for vectorized TX.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
+		      uint16_t pkts_n)
+{
+	struct txq *txq = (struct txq *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint16_t n;
+		uint16_t ret;
+
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, 0);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * DPDK callback for vectorized TX with multi-seg packets and offload.
+ *
+ * @param dpdk_txq
+ *   Generic pointer to TX queue structure.
+ * @param[in] pkts
+ *   Packets to transmit.
+ * @param pkts_n
+ *   Number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully transmitted (<= pkts_n).
+ */
+uint16_t
+mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct txq *txq = (struct txq *)dpdk_txq;
+	uint16_t nb_tx = 0;
+
+	while (pkts_n > nb_tx) {
+		uint8_t cs_flags = 0;
+		uint16_t n;
+		uint16_t ret;
+
+		/* Transmit multi-seg packets in the head of pkts list. */
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) &&
+		    NB_SEGS(pkts[nb_tx]) > 1)
+			nb_tx += txq_scatter_v(txq,
+					       &pkts[nb_tx],
+					       pkts_n - nb_tx);
+		n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST);
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS))
+			n = txq_check_multiseg(&pkts[nb_tx], n);
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
+			n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags);
+		ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags);
+		nb_tx += ret;
+		if (!ret)
+			break;
+	}
+	return nb_tx;
+}
+
+/**
+ * Store free buffers to RX SW ring.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param pkts
+ *   Pointer to array of packets to be stored.
+ * @param pkts_n
+ *   Number of packets to be stored.
+ */
+static inline void
+rxq_copy_mbuf_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t n)
+{
+	const uint16_t q_mask = (1 << rxq->elts_n) - 1;
+	struct rte_mbuf **elts = &(*rxq->elts)[rxq->rq_pi & q_mask];
+	unsigned int pos;
+	uint16_t p = n & -2;
+
+	for (pos = 0; pos < p; pos += 2) {
+		__m128i mbp;
+
+		mbp = _mm_loadu_si128((__m128i *)&elts[pos]);
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp);
+	}
+	if (n & 1)
+		pkts[pos] = elts[pos];
+}
+
+/**
+ * Replenish buffers for RX in bulk.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param n
+ *   Number of buffers to be replenished.
+ */
+static inline void
+rxq_replenish_bulk_mbuf(struct rxq *rxq, uint16_t n)
+{
+	const uint16_t q_n = 1 << rxq->elts_n;
+	const uint16_t q_mask = q_n - 1;
+	const uint16_t elts_idx = rxq->rq_ci & q_mask;
+	struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
+	volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx];
+	unsigned int i;
+
+	assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH);
+	assert(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
+	assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
+	if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
+		rxq->stats.rx_nombuf += n;
+		return;
+	}
+	for (i = 0; i < n; ++i)
+		wq[i].addr = htonll(rte_pktmbuf_mtod(elts[i], uintptr_t));
+	rxq->rq_ci += n;
+	rte_wmb();
+	*rxq->rq_db = htonl(rxq->rq_ci);
+}
+
+/**
+ * Decompress a compressed completion and fill in mbufs in RX SW ring with data
+ * extracted from the title completion descriptor.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cq
+ *   Pointer to completion array having a compressed completion at first.
+ * @param elts
+ *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
+ *   the title completion descriptor to be copied to the rest of mbufs.
+ */
+static inline void
+rxq_cq_decompress_v(struct rxq *rxq,
+		    volatile struct mlx5_cqe *cq,
+		    struct rte_mbuf **elts)
+{
+	volatile struct mlx5_mini_cqe8 *mcq = (void *)(cq + 1);
+	struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */
+	unsigned int pos;
+	unsigned int i;
+	unsigned int inv = 0;
+	/* Mask to shuffle from extracted mini CQE to mbuf. */
+	const __m128i shuf_mask1 =
+		_mm_set_epi8(0,  1,  2,  3, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			     6,  7,         /* data_len, bswap16 */
+			    -1, -1,  6,  7, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	const __m128i shuf_mask2 =
+		_mm_set_epi8(8,  9, 10, 11, /* rss, bswap32 */
+			    -1, -1,         /* skip vlan_tci */
+			    14, 15,         /* data_len, bswap16 */
+			    -1, -1, 14, 15, /* pkt_len, bswap16 */
+			    -1, -1, -1, -1  /* skip packet_type */);
+	/* Restore the compressed count. Must be 16 bits. */
+	const uint16_t mcqe_n = t_pkt->data_len +
+				(rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i rearm =
+		_mm_loadu_si128((__m128i *)&t_pkt->rearm_data);
+	const __m128i rxdf =
+		_mm_loadu_si128((__m128i *)&t_pkt->rx_descriptor_fields1);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0, 0);
+	const uint32_t flow_tag = t_pkt->hash.fdir.hi;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     14, 15,  6,  7,
+			     10, 11,  2,  3);
+#endif
+
+	/* Compile time sanity check for this function. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
+	/*
+	 * A. load mCQEs into a 128bit register.
+	 * B. store rearm data to mbuf.
+	 * C. combine data from mCQEs with rx_descriptor_fields1.
+	 * D. store rx_descriptor_fields1.
+	 * E. store flow tag (rte_flow mark).
+	 */
+	for (pos = 0; pos < mcqe_n; ) {
+		__m128i mcqe1, mcqe2;
+		__m128i rxdf1, rxdf2;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt, invalid_mask;
+#endif
+
+		if (!(pos & 0x7) && pos + 8 < mcqe_n)
+			rte_prefetch0((void *)(cq + pos + 8));
+		/* A.1 load mCQEs into a 128bit register. */
+		mcqe1 = _mm_loadu_si128((__m128i *)&mcq[pos % 8]);
+		mcqe2 = _mm_loadu_si128((__m128i *)&mcq[pos % 8 + 2]);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 1]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe1, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe1, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 1]->rx_descriptor_fields1,
+				 rxdf2);
+		/* B.1 store rearm data to mbuf. */
+		_mm_storeu_si128((__m128i *)&elts[pos + 2]->rearm_data, rearm);
+		_mm_storeu_si128((__m128i *)&elts[pos + 3]->rearm_data, rearm);
+		/* C.1 combine data from mCQEs with rx_descriptor_fields1. */
+		rxdf1 = _mm_shuffle_epi8(mcqe2, shuf_mask1);
+		rxdf2 = _mm_shuffle_epi8(mcqe2, shuf_mask2);
+		rxdf1 = _mm_sub_epi16(rxdf1, crc_adj);
+		rxdf2 = _mm_sub_epi16(rxdf2, crc_adj);
+		rxdf1 = _mm_blend_epi16(rxdf1, rxdf, 0x23);
+		rxdf2 = _mm_blend_epi16(rxdf2, rxdf, 0x23);
+		/* D.1 store rx_descriptor_fields1. */
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 2]->rx_descriptor_fields1,
+				 rxdf1);
+		_mm_storeu_si128((__m128i *)
+				  &elts[pos + 3]->rx_descriptor_fields1,
+				 rxdf2);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		invalid_mask = _mm_set_epi64x(0,
+					      (mcqe_n - pos) *
+					      sizeof(uint16_t) * 8);
+		invalid_mask = _mm_sll_epi64(ones, invalid_mask);
+		mcqe1 = _mm_srli_si128(mcqe1, 4);
+		byte_cnt = _mm_blend_epi16(mcqe1, mcqe2, 0xcc);
+		byte_cnt = _mm_shuffle_epi8(byte_cnt, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		if (rxq->mark) {
+			/* E.1 store flow tag (rte_flow mark). */
+			elts[pos]->hash.fdir.hi = flow_tag;
+			elts[pos + 1]->hash.fdir.hi = flow_tag;
+			elts[pos + 2]->hash.fdir.hi = flow_tag;
+			elts[pos + 3]->hash.fdir.hi = flow_tag;
+		}
+		pos += MLX5_VPMD_DESCS_PER_LOOP;
+		/* Move to next CQE and invalidate consumed CQEs. */
+		if (!(pos & 0x7) && pos < mcqe_n) {
+			mcq = (void *)(cq + pos);
+			for (i = 0; i < 8; ++i)
+				cq[inv++].op_own = MLX5_CQE_INVALIDATE;
+		}
+	}
+	/* Invalidate the rest of CQEs. */
+	for (; inv < mcqe_n; ++inv)
+		cq[inv].op_own = MLX5_CQE_INVALIDATE;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += mcqe_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	rxq->cq_ci += mcqe_n;
+}
+
+/**
+ * Calculate packet type and offload flag for mbuf and store it.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param cqes[4]
+ *   Array of four 16bytes completions extracted from the original completion
+ *   descriptor.
+ * @param op_err
+ *   Opcode vector having responder error status. Each field is 4B.
+ * @param pkts
+ *   Pointer to array of packets to be filled.
+ */
+static inline void
+rxq_cq_to_ptype_oflags_v(struct rxq *rxq, __m128i cqes[4], __m128i op_err,
+			 struct rte_mbuf **pkts)
+{
+	__m128i pinfo0, pinfo1;
+	__m128i pinfo, ptype;
+	__m128i ol_flags = _mm_set1_epi32(rxq->rss_hash * PKT_RX_RSS_HASH);
+	__m128i cv_flags;
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ptype_mask =
+		_mm_set_epi32(0xd06, 0xd06, 0xd06, 0xd06);
+	const __m128i ptype_ol_mask =
+		_mm_set_epi32(0x106, 0x106, 0x106, 0x106);
+	const __m128i pinfo_mask =
+		_mm_set_epi32(0x3, 0x3, 0x3, 0x3);
+	const __m128i cv_flag_sel =
+		_mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0,
+			     (uint8_t)((PKT_RX_IP_CKSUM_GOOD |
+					PKT_RX_L4_CKSUM_GOOD) >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1),
+			     0,
+			     (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1),
+			     (uint8_t)(PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED),
+			     0);
+	const __m128i cv_mask =
+		_mm_set_epi32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
+			      PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
+			      PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED);
+	const __m128i mbuf_init =
+		_mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer);
+	__m128i rearm0, rearm1, rearm2, rearm3;
+
+	/* Extract pkt_info field. */
+	pinfo0 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+	pinfo = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	/* Extract hdr_type_etc field. */
+	pinfo0 = _mm_unpackhi_epi32(cqes[0], cqes[1]);
+	pinfo1 = _mm_unpackhi_epi32(cqes[2], cqes[3]);
+	ptype = _mm_unpacklo_epi64(pinfo0, pinfo1);
+	if (rxq->mark) {
+		const __m128i pinfo_ft_mask =
+			_mm_set_epi32(0xffffff00, 0xffffff00,
+				      0xffffff00, 0xffffff00);
+		const __m128i fdir_flags = _mm_set1_epi32(PKT_RX_FDIR);
+		const __m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID);
+		__m128i flow_tag, invalid_mask;
+
+		flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask);
+		/* Check if flow tag is non-zero then set PKT_RX_FDIR. */
+		invalid_mask = _mm_cmpeq_epi32(flow_tag, zero);
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(invalid_mask,
+							 fdir_flags));
+		/* Mask out invalid entries. */
+		flow_tag = _mm_andnot_si128(invalid_mask, flow_tag);
+		/* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
+		ol_flags = _mm_or_si128(ol_flags,
+					_mm_andnot_si128(
+						_mm_cmpeq_epi32(flow_tag,
+								pinfo_ft_mask),
+						fdir_id_flags));
+	}
+	/*
+	 * Merge the two fields to generate the following:
+	 * bit[1]  = l3_ok,    bit[2]     = l4_ok
+	 * bit[8]  = cv,       bit[11:10] = l3_hdr_type
+	 * bit[12] = tunneled, bit[13]    = outer_l3_type
+	 */
+	ptype = _mm_and_si128(ptype, ptype_mask);
+	pinfo = _mm_and_si128(pinfo, pinfo_mask);
+	pinfo = _mm_slli_epi32(pinfo, 12);
+	ptype = _mm_or_si128(ptype, pinfo);
+	ptype = _mm_srli_epi32(ptype, 10);
+	ptype = _mm_packs_epi32(ptype, zero);
+	/* Errored packets will have RTE_PTYPE_ALL_MASK. */
+	op_err = _mm_srli_epi16(op_err, 12);
+	ptype = _mm_or_si128(ptype, op_err);
+	pkts[0]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 0)];
+	pkts[1]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 2)];
+	pkts[2]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 4)];
+	pkts[3]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 6)];
+	/* Fill flags for checksum and VLAN. */
+	pinfo = _mm_and_si128(pinfo, ptype_ol_mask);
+	pinfo = _mm_shuffle_epi8(cv_flag_sel, pinfo);
+	/* Locate checksum flags at byte[2:1] and merge with VLAN flags. */
+	cv_flags = _mm_slli_epi32(pinfo, 9);
+	cv_flags = _mm_or_si128(pinfo, cv_flags);
+	/* Move back flags to start from byte[0]. */
+	cv_flags = _mm_srli_epi32(cv_flags, 8);
+	/* Mask out garbage bits. */
+	cv_flags = _mm_and_si128(cv_flags, cv_mask);
+	/* Merge to ol_flags. */
+	ol_flags = _mm_or_si128(ol_flags, cv_flags);
+	/* Merge mbuf_init and ol_flags. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+			 offsetof(struct rte_mbuf, rearm_data) + 8);
+	rearm0 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 8), 0x30);
+	rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(ol_flags, 4), 0x30);
+	rearm2 = _mm_blend_epi16(mbuf_init, ol_flags, 0x30);
+	rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(ol_flags, 4), 0x30);
+	/* Write 8B rearm_data and 8B ol_flags. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+			 RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
+	_mm_store_si128((__m128i *)&pkts[0]->rearm_data, rearm0);
+	_mm_store_si128((__m128i *)&pkts[1]->rearm_data, rearm1);
+	_mm_store_si128((__m128i *)&pkts[2]->rearm_data, rearm2);
+	_mm_store_si128((__m128i *)&pkts[3]->rearm_data, rearm3);
+}
+
+/**
+ * Skip error packets.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+static uint16_t
+rxq_handle_pending_error(struct rxq *rxq, struct rte_mbuf **pkts,
+			 uint16_t pkts_n)
+{
+	uint16_t n = 0;
+	unsigned int i;
+
+	for (i = 0; i < pkts_n; ++i) {
+		struct rte_mbuf *pkt = pkts[i];
+
+		if (pkt->packet_type == RTE_PTYPE_ALL_MASK)
+			rte_pktmbuf_free_seg(pkt);
+		else
+			pkts[n++] = pkt;
+	}
+	rxq->stats.idropped += (pkts_n - n);
+	rxq->pending_err = 0;
+	return n;
+}
+
+/**
+ * Receive burst of packets. An errored completion also consumes a mbuf, but the
+ * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed
+ * before returning to application.
+ *
+ * @param rxq
+ *   Pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets received including errors (<= pkts_n).
+ */
+static inline uint16_t
+rxq_burst_v(struct rxq *rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	const uint16_t q_n = 1 << rxq->cqe_n;
+	const uint16_t q_mask = q_n - 1;
+	volatile struct mlx5_cqe *cq;
+	struct rte_mbuf **elts;
+	unsigned int pos;
+	uint64_t n;
+	uint16_t repl_n;
+	uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
+	uint16_t nocmp_n = 0;
+	uint16_t rcvd_pkt = 0;
+	unsigned int cq_idx = rxq->cq_ci & q_mask;
+	unsigned int elts_idx;
+	unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1));
+	const __m128i owner_check =
+		_mm_set_epi64x(0x0100000001000000LL, 0x0100000001000000LL);
+	const __m128i opcode_check =
+		_mm_set_epi64x(0xf0000000f0000000LL, 0xf0000000f0000000LL);
+	const __m128i format_check =
+		_mm_set_epi64x(0x0c0000000c000000LL, 0x0c0000000c000000LL);
+	const __m128i resp_err_check =
+		_mm_set_epi64x(0xe0000000e0000000LL, 0xe0000000e0000000LL);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint32_t rcvd_byte = 0;
+	/* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
+	const __m128i len_shuf_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			     12, 13,  8,  9,
+			      4,  5,  0,  1);
+#endif
+	/* Mask to shuffle from extracted CQE to mbuf. */
+	const __m128i shuf_mask =
+		_mm_set_epi8(-1,  3,  2,  1, /* fdir.hi */
+			     12, 13, 14, 15, /* rss, bswap32 */
+			     10, 11,         /* vlan_tci, bswap16 */
+			      4,  5,         /* data_len, bswap16 */
+			     -1, -1,         /* zero out 2nd half of pkt_len */
+			      4,  5          /* pkt_len, bswap16 */);
+	/* Mask to blend from the last Qword to the first DQword. */
+	const __m128i blend_mask =
+		_mm_set_epi8(-1, -1, -1, -1,
+			     -1, -1, -1, -1,
+			      0,  0,  0,  0,
+			      0,  0,  0, -1);
+	const __m128i zero = _mm_setzero_si128();
+	const __m128i ones = _mm_cmpeq_epi32(zero, zero);
+	const __m128i crc_adj =
+		_mm_set_epi16(0, 0, 0, 0, 0,
+			      rxq->crc_present * ETHER_CRC_LEN,
+			      0,
+			      rxq->crc_present * ETHER_CRC_LEN);
+	const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0);
+
+	/* Compile time sanity check for this function. */
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+			 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, pkt_info) != 0);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rx_hash_res) !=
+			 offsetof(struct mlx5_cqe, pkt_info) + 12);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd1) +
+			  sizeof(((struct mlx5_cqe *)0)->rsvd1) !=
+			 offsetof(struct mlx5_cqe, hdr_type_etc));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, vlan_info) !=
+			 offsetof(struct mlx5_cqe, hdr_type_etc) + 2);
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, rsvd2) +
+			  sizeof(((struct mlx5_cqe *)0)->rsvd2) !=
+			 offsetof(struct mlx5_cqe, byte_cnt));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, sop_drop_qpn) !=
+			 RTE_ALIGN(offsetof(struct mlx5_cqe, sop_drop_qpn), 8));
+	RTE_BUILD_BUG_ON(offsetof(struct mlx5_cqe, op_own) !=
+			 offsetof(struct mlx5_cqe, sop_drop_qpn) + 7);
+	assert(rxq->sges_n == 0);
+	assert(rxq->cqe_n == rxq->elts_n);
+	cq = &(*rxq->cqes)[cq_idx];
+	rte_prefetch0(cq);
+	rte_prefetch0(cq + 1);
+	rte_prefetch0(cq + 2);
+	rte_prefetch0(cq + 3);
+	pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
+	/*
+	 * Order of indexes:
+	 *   rq_ci >= cq_ci >= rq_pi
+	 * Definition of indexes:
+	 *   rq_ci - cq_ci := # of buffers owned by HW (posted).
+	 *   cq_ci - rq_pi := # of buffers not returned to app (decompressed).
+	 *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
+	 */
+	repl_n = q_n - (rxq->rq_ci - rxq->rq_pi);
+	if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH)
+		rxq_replenish_bulk_mbuf(rxq, repl_n);
+	/* See if there're unreturned mbufs from compressed CQE. */
+	rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
+	if (rcvd_pkt > 0) {
+		rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
+		rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt);
+		rxq->rq_pi += rcvd_pkt;
+		pkts += rcvd_pkt;
+	}
+	elts_idx = rxq->rq_pi & q_mask;
+	elts = &(*rxq->elts)[elts_idx];
+	/* Not to overflow pkts array. */
+	pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP);
+	/* Not to cross queue end. */
+	pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
+	if (!pkts_n)
+		return rcvd_pkt;
+	/* At this point, there shouldn't be any remained packets. */
+	assert(rxq->rq_pi == rxq->cq_ci);
+	/*
+	 * A. load first Qword (8bytes) in one loop.
+	 * B. copy 4 mbuf pointers from elts ring to returing pkts.
+	 * C. load remained CQE data and extract necessary fields.
+	 *    Final 16bytes cqes[] extracted from original 64bytes CQE has the
+	 *    following structure:
+	 *        struct {
+	 *          uint8_t  pkt_info;
+	 *          uint8_t  flow_tag[3];
+	 *          uint16_t byte_cnt;
+	 *          uint8_t  rsvd4;
+	 *          uint8_t  op_own;
+	 *          uint16_t hdr_type_etc;
+	 *          uint16_t vlan_info;
+	 *          uint32_t rx_has_res;
+	 *        } c;
+	 * D. fill in mbuf.
+	 * E. get valid CQEs.
+	 * F. find compressed CQE.
+	 */
+	for (pos = 0;
+	     pos < pkts_n;
+	     pos += MLX5_VPMD_DESCS_PER_LOOP) {
+		__m128i cqes[MLX5_VPMD_DESCS_PER_LOOP];
+		__m128i cqe_tmp1, cqe_tmp2;
+		__m128i pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
+		__m128i op_own, op_own_tmp1, op_own_tmp2;
+		__m128i opcode, owner_mask, invalid_mask;
+		__m128i comp_mask;
+		__m128i mask;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		__m128i byte_cnt;
+#endif
+		__m128i mbp1, mbp2;
+		__m128i p = _mm_set_epi16(0, 0, 0, 0, 3, 2, 1, 0);
+		unsigned int p1, p2, p3;
+
+		/* Prefetch next 4 CQEs. */
+		if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) {
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]);
+			rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]);
+		}
+		/* A.0 do not cross the end of CQ. */
+		mask = _mm_set_epi64x(0, (pkts_n - pos) * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		p = _mm_andnot_si128(mask, p);
+		/* A.1 load cqes. */
+		p3 = _mm_extract_epi16(p, 3);
+		cqes[3] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p3].sop_drop_qpn);
+		rte_compiler_barrier();
+		p2 = _mm_extract_epi16(p, 2);
+		cqes[2] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p2].sop_drop_qpn);
+		rte_compiler_barrier();
+		/* B.1 load mbuf pointers. */
+		mbp1 = _mm_loadu_si128((__m128i *)&elts[pos]);
+		mbp2 = _mm_loadu_si128((__m128i *)&elts[pos + 2]);
+		/* A.1 load a block having op_own. */
+		p1 = _mm_extract_epi16(p, 1);
+		cqes[1] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos + p1].sop_drop_qpn);
+		rte_compiler_barrier();
+		cqes[0] = _mm_loadl_epi64((__m128i *)
+					   &cq[pos].sop_drop_qpn);
+		/* B.2 copy mbuf pointers. */
+		_mm_storeu_si128((__m128i *)&pkts[pos], mbp1);
+		_mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2);
+		rte_compiler_barrier();
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]);
+		cqes[3] = _mm_blendv_epi8(cqes[3], cqe_tmp2, blend_mask);
+		cqes[2] = _mm_blendv_epi8(cqes[2], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p3].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos + p2].rsvd1[3]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x30);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p3].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos + p2].rsvd2[10]);
+		cqes[3] = _mm_blend_epi16(cqes[3], cqe_tmp2, 0x04);
+		cqes[2] = _mm_blend_epi16(cqes[2], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb3 = _mm_shuffle_epi8(cqes[3], shuf_mask);
+		pkt_mb2 = _mm_shuffle_epi8(cqes[2], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb3 = _mm_sub_epi16(pkt_mb3, crc_adj);
+		pkt_mb2 = _mm_sub_epi16(pkt_mb2, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb3 = _mm_add_epi32(pkt_mb3, flow_mark_adj);
+		pkt_mb2 = _mm_add_epi32(pkt_mb2, flow_mark_adj);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 3]->pkt_len, pkt_mb3);
+		_mm_storeu_si128((void *)&pkts[pos + 2]->pkt_len, pkt_mb2);
+		/* E.1 extract op_own field. */
+		op_own_tmp2 = _mm_unpacklo_epi32(cqes[2], cqes[3]);
+		/* C.1 load remained CQE data and extract necessary fields. */
+		cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p1]);
+		cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos]);
+		cqes[1] = _mm_blendv_epi8(cqes[1], cqe_tmp2, blend_mask);
+		cqes[0] = _mm_blendv_epi8(cqes[0], cqe_tmp1, blend_mask);
+		cqe_tmp2 = _mm_loadu_si128((__m128i *)&cq[pos + p1].rsvd1[3]);
+		cqe_tmp1 = _mm_loadu_si128((__m128i *)&cq[pos].rsvd1[3]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x30);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x30);
+		cqe_tmp2 = _mm_loadl_epi64((__m128i *)&cq[pos + p1].rsvd2[10]);
+		cqe_tmp1 = _mm_loadl_epi64((__m128i *)&cq[pos].rsvd2[10]);
+		cqes[1] = _mm_blend_epi16(cqes[1], cqe_tmp2, 0x04);
+		cqes[0] = _mm_blend_epi16(cqes[0], cqe_tmp1, 0x04);
+		/* C.2 generate final structure for mbuf with swapping bytes. */
+		pkt_mb1 = _mm_shuffle_epi8(cqes[1], shuf_mask);
+		pkt_mb0 = _mm_shuffle_epi8(cqes[0], shuf_mask);
+		/* C.3 adjust CRC length. */
+		pkt_mb1 = _mm_sub_epi16(pkt_mb1, crc_adj);
+		pkt_mb0 = _mm_sub_epi16(pkt_mb0, crc_adj);
+		/* C.4 adjust flow mark. */
+		pkt_mb1 = _mm_add_epi32(pkt_mb1, flow_mark_adj);
+		pkt_mb0 = _mm_add_epi32(pkt_mb0, flow_mark_adj);
+		/* E.1 extract op_own byte. */
+		op_own_tmp1 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
+		op_own = _mm_unpackhi_epi64(op_own_tmp1, op_own_tmp2);
+		/* D.1 fill in mbuf - rx_descriptor_fields1. */
+		_mm_storeu_si128((void *)&pkts[pos + 1]->pkt_len, pkt_mb1);
+		_mm_storeu_si128((void *)&pkts[pos]->pkt_len, pkt_mb0);
+		/* E.2 flip owner bit to mark CQEs from last round. */
+		owner_mask = _mm_and_si128(op_own, owner_check);
+		if (ownership)
+			owner_mask = _mm_xor_si128(owner_mask, owner_check);
+		owner_mask = _mm_cmpeq_epi32(owner_mask, owner_check);
+		owner_mask = _mm_packs_epi32(owner_mask, zero);
+		/* E.3 get mask for invalidated CQEs. */
+		opcode = _mm_and_si128(op_own, opcode_check);
+		invalid_mask = _mm_cmpeq_epi32(opcode_check, opcode);
+		invalid_mask = _mm_packs_epi32(invalid_mask, zero);
+		/* E.4 mask out beyond boundary. */
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* E.5 merge invalid_mask with invalid owner. */
+		invalid_mask = _mm_or_si128(invalid_mask, owner_mask);
+		/* F.1 find compressed CQE format. */
+		comp_mask = _mm_and_si128(op_own, format_check);
+		comp_mask = _mm_cmpeq_epi32(comp_mask, format_check);
+		comp_mask = _mm_packs_epi32(comp_mask, zero);
+		/* F.2 mask out invalid entries. */
+		comp_mask = _mm_andnot_si128(invalid_mask, comp_mask);
+		comp_idx = _mm_cvtsi128_si64(comp_mask);
+		/* F.3 get the first compressed CQE. */
+		comp_idx = comp_idx ?
+				__builtin_ctzll(comp_idx) /
+					(sizeof(uint16_t) * 8) :
+				MLX5_VPMD_DESCS_PER_LOOP;
+		/* E.6 mask out entries after the compressed CQE. */
+		mask = _mm_set_epi64x(0, comp_idx * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* E.7 count non-compressed valid CQEs. */
+		n = _mm_cvtsi128_si64(invalid_mask);
+		n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) :
+			MLX5_VPMD_DESCS_PER_LOOP;
+		nocmp_n += n;
+		/* D.2 get the final invalid mask. */
+		mask = _mm_set_epi64x(0, n * sizeof(uint16_t) * 8);
+		mask = _mm_sll_epi64(ones, mask);
+		invalid_mask = _mm_or_si128(invalid_mask, mask);
+		/* D.3 check error in opcode. */
+		opcode = _mm_cmpeq_epi32(resp_err_check, opcode);
+		opcode = _mm_packs_epi32(opcode, zero);
+		opcode = _mm_andnot_si128(invalid_mask, opcode);
+		/* D.4 mark if any error is set */
+		rxq->pending_err |= !!_mm_cvtsi128_si64(opcode);
+		/* D.5 fill in mbuf - rearm_data and packet_type. */
+		rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Add up received bytes count. */
+		byte_cnt = _mm_shuffle_epi8(op_own, len_shuf_mask);
+		byte_cnt = _mm_andnot_si128(invalid_mask, byte_cnt);
+		byte_cnt = _mm_hadd_epi16(byte_cnt, zero);
+		rcvd_byte += _mm_cvtsi128_si64(_mm_hadd_epi16(byte_cnt, zero));
+#endif
+		/*
+		 * Break the loop unless more valid CQE is expected, or if
+		 * there's a compressed CQE.
+		 */
+		if (n != MLX5_VPMD_DESCS_PER_LOOP)
+			break;
+	}
+	/* If no new CQE seen, return without updating cq_db. */
+	if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP))
+		return rcvd_pkt;
+	/* Update the consumer indexes for non-compressed CQEs. */
+	assert(nocmp_n <= pkts_n);
+	rxq->cq_ci += nocmp_n;
+	rxq->rq_pi += nocmp_n;
+	rcvd_pkt += nocmp_n;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	rxq->stats.ipackets += nocmp_n;
+	rxq->stats.ibytes += rcvd_byte;
+#endif
+	/* Decompress the last CQE if compressed. */
+	if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) {
+		assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
+		rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]);
+		/* Return more packets if needed. */
+		if (nocmp_n < pkts_n) {
+			uint16_t n = rxq->cq_ci - rxq->rq_pi;
+
+			n = RTE_MIN(n, pkts_n - nocmp_n);
+			rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n);
+			rxq->rq_pi += n;
+			rcvd_pkt += n;
+		}
+	}
+	rte_wmb();
+	*rxq->cq_db = htonl(rxq->cq_ci);
+	return rcvd_pkt;
+}
+
+/**
+ * DPDK callback for vectorized RX.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct rxq *rxq = dpdk_rxq;
+	uint16_t nb_rx;
+
+	nb_rx = rxq_burst_v(rxq, pkts, pkts_n);
+	if (unlikely(rxq->pending_err))
+		nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx);
+	return nb_rx;
+}
+
+/**
+ * Check Tx queue flags are set for raw vectorized Tx.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_raw_vec_tx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->txqs_n; ++i) {
+		struct txq *txq = (*priv->txqs)[i];
+
+		if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) ||
+		    !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS))
+			break;
+	}
+	if (i != priv->txqs_n)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized TX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_vec_tx_support(struct priv *priv)
+{
+	if (priv->txqs_n > MLX5_VPMD_MIN_TXQS ||
+	    priv->mps != MLX5_MPW_ENHANCED ||
+	    priv->tso)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a RX queue can support vectorized RX.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+rxq_check_vec_support(struct rxq *rxq)
+{
+	if (rxq->sges_n != 0)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Check a device can support vectorized RX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+int __attribute__((cold))
+priv_check_vec_rx_support(struct priv *priv)
+{
+	uint16_t i;
+
+	/* All the configured queues should support. */
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+
+		if (rxq_check_vec_support(rxq) < 0)
+			break;
+	}
+	if (i != priv->rxqs_n)
+		return -ENOTSUP;
+	return 1;
+}
+
+/**
+ * Prepare for vectorized RX.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+void
+priv_prep_vec_rx_function(struct priv *priv)
+{
+	uint16_t i;
+
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct rxq *rxq = (*priv->rxqs)[i];
+		struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
+		const uint16_t desc = 1 << rxq->elts_n;
+		int j;
+
+		assert(rxq->elts_n == rxq->cqe_n);
+		/* Initialize default rearm_data for vPMD. */
+		mbuf_init->data_off = RTE_PKTMBUF_HEADROOM;
+		rte_mbuf_refcnt_set(mbuf_init, 1);
+		mbuf_init->nb_segs = 1;
+		mbuf_init->port = rxq->port_id;
+		/*
+		 * prevent compiler reordering:
+		 * rearm_data covers previous fields.
+		 */
+		rte_compiler_barrier();
+		rxq->mbuf_initializer =
+			*(uint64_t *)&mbuf_init->rearm_data;
+		/* Padding with a fake mbuf for vectorized Rx. */
+		for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j)
+			(*rxq->elts)[desc + j] = &rxq->fake_mbuf;
+		/* Mark that it need to be cleaned up for rxq_alloc_elts(). */
+		rxq->trim_elts = 1;
+	}
+}
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 03e23f58d..3ad6ac0c6 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -243,7 +243,7 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
 		ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set");
 		goto error;
 	}
-	(void)conf; /* Thresholds configuration (ignored). */
+	tmpl.txq.flags = conf->txq_flags;
 	assert(desc > MLX5_TX_COMP_THRESH);
 	tmpl.txq.elts_n = log2above(desc);
 	if (priv->mps == MLX5_MPW_ENHANCED)
-- 
2.11.0

Re: [dpdk-dev] [PATCH v4 0/5] net/mlx5: add vectorized Rx/Tx burst for x86

[Ferruh Yigit]

On 7/6/2017 7:41 PM, Yongseok Koh wrote:
> This is to introduce more efficient Rx/Tx burst functions using SIMD
> instructions. Currently it is only supported by 64bit x86 having SSE4.1.
> 
> From functional perspective, Rx burst function is equivalent to the
> existing mlx5_rx_burst() except for scatter support, which will be added
> soon. Tx burst function supports multi-segment packets and offload flags
> unless it is disabled by txq_flags. However, disabling those features will
> bring a little higher performance.
> 
> v4:
> * Restore 'pragma' for PEDANTIC.
> * Resolve compile error with clang.
> 
> v3:
> * Remove requirement of SSE4.1 as DPDK now mandates SSE4.2 support at least.
> * Bug fix in "net/mlx5: select Rx/Tx callbacks when starting device"
>   - Need to re-select Rx burst func when chaning MTU size.
> * Resolved an optimization issue of gcc-6 in rxq_burst_v()
>   - Bit shift (<<) for 128b vector type is compiled differently. 'psllq' is
>     needed instead of 'sal'.
> * Minor changes to address what is mentioned by review.
>   - Remove 'pragma' for PEDANTIC
>   - Make mlx5_ptype_table global.
>   - Change name of some inline funcs which also exist in mlx4 by the same name.
>   - Fix comments and indentation/spacing.
> 
> v2:
> * Streamline redundant conditional clauses in txq_complete().
> * Remove the mempool pointer in txq->mp2mr structure.
> * Fix indentation and spacing.
> 
> Yongseok Koh (5):
>   net/mlx5: change indexing for Tx SW ring
>   net/mlx5: free buffers in bulk on Tx completion
>   net/mlx5: use buffer address for LKEY search
>   net/mlx5: select Rx/Tx callbacks when starting device
>   net/mlx5: add vectorized Rx/Tx burst for x86

Series applied to dpdk-next-net/master, thanks.