[dpdk-dev] [PATCH 0/5] i40e: vector poll-mode driver on ARM64

[dpdk-dev] [PATCH 0/5] i40e: vector poll-mode driver on ARM64

[Jianbo Liu]

This patch set is to implement i40e vector PMD on ARM64.
For x86, vPMD is only reorganized, there should be no performance loss.

Jianbo Liu (5):
  i40e: extract non-x86 specific code from vector driver
  i40e: implement vector PMD for ARM architecture
  i40e: enable i40e vector PMD on ARMv8a platform
  i40e: make vector driver filenames consistent
  maintainers: claim i40e vector PMD on ARM

 MAINTAINERS                                        |   1 +
 config/defconfig_arm64-armv8a-linuxapp-gcc         |   1 -
 doc/guides/nics/features/i40e_vec.ini              |   1 +
 doc/guides/nics/features/i40e_vf_vec.ini           |   1 +
 drivers/net/i40e/Makefile                          |   8 +-
 drivers/net/i40e/i40e_rxtx_vec_common.h            | 239 +++++++++
 drivers/net/i40e/i40e_rxtx_vec_neon.c              | 581 +++++++++++++++++++++
 .../i40e/{i40e_rxtx_vec.c => i40e_rxtx_vec_sse.c}  | 184 +------
 8 files changed, 833 insertions(+), 183 deletions(-)
 create mode 100644 drivers/net/i40e/i40e_rxtx_vec_common.h
 create mode 100644 drivers/net/i40e/i40e_rxtx_vec_neon.c
 rename drivers/net/i40e/{i40e_rxtx_vec.c => i40e_rxtx_vec_sse.c} (78%)

-- 
2.4.11

[dpdk-dev] [PATCH 1/5] i40e: extract non-x86 specific code from vector driver

[Jianbo Liu]

move scalar code which does not use x86 intrinsic functions to new file
"i40e_rxtx_vec_common.h", while keeping x86 code in i40e_rxtx_vec.c.
This allows the scalar code to to be shared among vector drivers for
different platforms.

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 drivers/net/i40e/i40e_rxtx_vec.c        | 184 +-----------------------
 drivers/net/i40e/i40e_rxtx_vec_common.h | 239 ++++++++++++++++++++++++++++++++
 2 files changed, 243 insertions(+), 180 deletions(-)
 create mode 100644 drivers/net/i40e/i40e_rxtx_vec_common.h

diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec.c
index 51fb282..f847469 100644
--- a/drivers/net/i40e/i40e_rxtx_vec.c
+++ b/drivers/net/i40e/i40e_rxtx_vec.c
@@ -39,6 +39,7 @@
 #include "base/i40e_type.h"
 #include "i40e_ethdev.h"
 #include "i40e_rxtx.h"
+#include "i40e_rxtx_vec_common.h"
 
 #include <tmmintrin.h>
 
@@ -421,68 +422,6 @@ i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 	return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
 }
 
-static inline uint16_t
-reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
-		   uint16_t nb_bufs, uint8_t *split_flags)
-{
-	struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/
-	struct rte_mbuf *start = rxq->pkt_first_seg;
-	struct rte_mbuf *end =  rxq->pkt_last_seg;
-	unsigned pkt_idx, buf_idx;
-
-	for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
-		if (end != NULL) {
-			/* processing a split packet */
-			end->next = rx_bufs[buf_idx];
-			rx_bufs[buf_idx]->data_len += rxq->crc_len;
-
-			start->nb_segs++;
-			start->pkt_len += rx_bufs[buf_idx]->data_len;
-			end = end->next;
-
-			if (!split_flags[buf_idx]) {
-				/* it's the last packet of the set */
-				start->hash = end->hash;
-				start->ol_flags = end->ol_flags;
-				/* we need to strip crc for the whole packet */
-				start->pkt_len -= rxq->crc_len;
-				if (end->data_len > rxq->crc_len) {
-					end->data_len -= rxq->crc_len;
-				} else {
-					/* free up last mbuf */
-					struct rte_mbuf *secondlast = start;
-
-					while (secondlast->next != end)
-						secondlast = secondlast->next;
-					secondlast->data_len -= (rxq->crc_len -
-							end->data_len);
-					secondlast->next = NULL;
-					rte_pktmbuf_free_seg(end);
-					end = secondlast;
-				}
-				pkts[pkt_idx++] = start;
-				start = end = NULL;
-			}
-		} else {
-			/* not processing a split packet */
-			if (!split_flags[buf_idx]) {
-				/* not a split packet, save and skip */
-				pkts[pkt_idx++] = rx_bufs[buf_idx];
-				continue;
-			}
-			end = start = rx_bufs[buf_idx];
-			rx_bufs[buf_idx]->data_len += rxq->crc_len;
-			rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
-		}
-	}
-
-	/* save the partial packet for next time */
-	rxq->pkt_first_seg = start;
-	rxq->pkt_last_seg = end;
-	memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
-	return pkt_idx;
-}
-
  /* vPMD receive routine that reassembles scattered packets
  * Notice:
  * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
@@ -548,73 +487,6 @@ vtx(volatile struct i40e_tx_desc *txdp,
 		vtx1(txdp, *pkt, flags);
 }
 
-static inline int __attribute__((always_inline))
-i40e_tx_free_bufs(struct i40e_tx_queue *txq)
-{
-	struct i40e_tx_entry *txep;
-	uint32_t n;
-	uint32_t i;
-	int nb_free = 0;
-	struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
-
-	/* check DD bits on threshold descriptor */
-	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
-			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
-			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
-		return 0;
-
-	n = txq->tx_rs_thresh;
-
-	 /* first buffer to free from S/W ring is at index
-	  * tx_next_dd - (tx_rs_thresh-1)
-	  */
-	txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
-	m = __rte_pktmbuf_prefree_seg(txep[0].mbuf);
-	if (likely(m != NULL)) {
-		free[0] = m;
-		nb_free = 1;
-		for (i = 1; i < n; i++) {
-			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
-			if (likely(m != NULL)) {
-				if (likely(m->pool == free[0]->pool)) {
-					free[nb_free++] = m;
-				} else {
-					rte_mempool_put_bulk(free[0]->pool,
-							     (void *)free,
-							     nb_free);
-					free[0] = m;
-					nb_free = 1;
-				}
-			}
-		}
-		rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
-	} else {
-		for (i = 1; i < n; i++) {
-			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
-			if (m != NULL)
-				rte_mempool_put(m->pool, m);
-		}
-	}
-
-	/* buffers were freed, update counters */
-	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
-	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
-	if (txq->tx_next_dd >= txq->nb_tx_desc)
-		txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
-
-	return txq->tx_rs_thresh;
-}
-
-static inline void __attribute__((always_inline))
-tx_backlog_entry(struct i40e_tx_entry *txep,
-		 struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
-{
-	int i;
-
-	for (i = 0; i < (int)nb_pkts; ++i)
-		txep[i].mbuf = tx_pkts[i];
-}
-
 uint16_t
 i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 		   uint16_t nb_pkts)
@@ -685,37 +557,13 @@ i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 void __attribute__((cold))
 i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)
 {
-	const unsigned mask = rxq->nb_rx_desc - 1;
-	unsigned i;
-
-	if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc)
-		return;
-
-	/* free all mbufs that are valid in the ring */
-	for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask)
-		rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
-	rxq->rxrearm_nb = rxq->nb_rx_desc;
-
-	/* set all entries to NULL */
-	memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
+	_i40e_rx_queue_release_mbufs_vec(rxq);
 }
 
 int __attribute__((cold))
 i40e_rxq_vec_setup(struct i40e_rx_queue *rxq)
 {
-	uintptr_t p;
-	struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
-
-	mb_def.nb_segs = 1;
-	mb_def.data_off = RTE_PKTMBUF_HEADROOM;
-	mb_def.port = rxq->port_id;
-	rte_mbuf_refcnt_set(&mb_def, 1);
-
-	/* prevent compiler reordering: rearm_data covers previous fields */
-	rte_compiler_barrier();
-	p = (uintptr_t)&mb_def.rearm_data;
-	rxq->mbuf_initializer = *(uint64_t *)p;
-	return 0;
+	return i40e_rxq_vec_setup_default(rxq);
 }
 
 int __attribute__((cold))
@@ -728,34 +576,10 @@ int __attribute__((cold))
 i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)
 {
 #ifndef RTE_LIBRTE_IEEE1588
-	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
-	struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf;
-
 	/* need SSE4.1 support */
 	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
 		return -1;
-
-#ifndef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
-	/* whithout rx ol_flags, no VP flag report */
-	if (rxmode->hw_vlan_strip != 0 ||
-	    rxmode->hw_vlan_extend != 0)
-		return -1;
 #endif
 
-	/* no fdir support */
-	if (fconf->mode != RTE_FDIR_MODE_NONE)
-		return -1;
-
-	 /* - no csum error report support
-	 * - no header split support
-	 */
-	if (rxmode->hw_ip_checksum == 1 ||
-	    rxmode->header_split == 1)
-		return -1;
-
-	return 0;
-#else
-	RTE_SET_USED(dev);
-	return -1;
-#endif
+	return i40e_rx_vec_dev_conf_condition_check_default(dev);
 }
diff --git a/drivers/net/i40e/i40e_rxtx_vec_common.h b/drivers/net/i40e/i40e_rxtx_vec_common.h
new file mode 100644
index 0000000..b31b39e
--- /dev/null
+++ b/drivers/net/i40e/i40e_rxtx_vec_common.h
@@ -0,0 +1,239 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _I40E_RXTX_VEC_COMMON_H_
+#define _I40E_RXTX_VEC_COMMON_H_
+#include <stdint.h>
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+
+#include "i40e_ethdev.h"
+#include "i40e_rxtx.h"
+
+static inline uint16_t
+reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
+		   uint16_t nb_bufs, uint8_t *split_flags)
+{
+	struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/
+	struct rte_mbuf *start = rxq->pkt_first_seg;
+	struct rte_mbuf *end =  rxq->pkt_last_seg;
+	unsigned pkt_idx, buf_idx;
+
+	for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
+		if (end != NULL) {
+			/* processing a split packet */
+			end->next = rx_bufs[buf_idx];
+			rx_bufs[buf_idx]->data_len += rxq->crc_len;
+
+			start->nb_segs++;
+			start->pkt_len += rx_bufs[buf_idx]->data_len;
+			end = end->next;
+
+			if (!split_flags[buf_idx]) {
+				/* it's the last packet of the set */
+				start->hash = end->hash;
+				start->ol_flags = end->ol_flags;
+				/* we need to strip crc for the whole packet */
+				start->pkt_len -= rxq->crc_len;
+				if (end->data_len > rxq->crc_len) {
+					end->data_len -= rxq->crc_len;
+				} else {
+					/* free up last mbuf */
+					struct rte_mbuf *secondlast = start;
+
+					while (secondlast->next != end)
+						secondlast = secondlast->next;
+					secondlast->data_len -= (rxq->crc_len -
+							end->data_len);
+					secondlast->next = NULL;
+					rte_pktmbuf_free_seg(end);
+					end = secondlast;
+				}
+				pkts[pkt_idx++] = start;
+				start = end = NULL;
+			}
+		} else {
+			/* not processing a split packet */
+			if (!split_flags[buf_idx]) {
+				/* not a split packet, save and skip */
+				pkts[pkt_idx++] = rx_bufs[buf_idx];
+				continue;
+			}
+			end = start = rx_bufs[buf_idx];
+			rx_bufs[buf_idx]->data_len += rxq->crc_len;
+			rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
+		}
+	}
+
+	/* save the partial packet for next time */
+	rxq->pkt_first_seg = start;
+	rxq->pkt_last_seg = end;
+	memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
+	return pkt_idx;
+}
+
+static inline int __attribute__((always_inline))
+i40e_tx_free_bufs(struct i40e_tx_queue *txq)
+{
+	struct i40e_tx_entry *txep;
+	uint32_t n;
+	uint32_t i;
+	int nb_free = 0;
+	struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
+
+	/* check DD bits on threshold descriptor */
+	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
+			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
+			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
+		return 0;
+
+	n = txq->tx_rs_thresh;
+
+	 /* first buffer to free from S/W ring is at index
+	  * tx_next_dd - (tx_rs_thresh-1)
+	  */
+	txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
+	m = __rte_pktmbuf_prefree_seg(txep[0].mbuf);
+	if (likely(m != NULL)) {
+		free[0] = m;
+		nb_free = 1;
+		for (i = 1; i < n; i++) {
+			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
+			if (likely(m != NULL)) {
+				if (likely(m->pool == free[0]->pool)) {
+					free[nb_free++] = m;
+				} else {
+					rte_mempool_put_bulk(free[0]->pool,
+							     (void *)free,
+							     nb_free);
+					free[0] = m;
+					nb_free = 1;
+				}
+			}
+		}
+		rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
+	} else {
+		for (i = 1; i < n; i++) {
+			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
+			if (m != NULL)
+				rte_mempool_put(m->pool, m);
+		}
+	}
+
+	/* buffers were freed, update counters */
+	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
+	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
+	if (txq->tx_next_dd >= txq->nb_tx_desc)
+		txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
+
+	return txq->tx_rs_thresh;
+}
+
+static inline void __attribute__((always_inline))
+tx_backlog_entry(struct i40e_tx_entry *txep,
+		 struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+	int i;
+
+	for (i = 0; i < (int)nb_pkts; ++i)
+		txep[i].mbuf = tx_pkts[i];
+}
+
+static inline void
+_i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)
+{
+	const unsigned mask = rxq->nb_rx_desc - 1;
+	unsigned i;
+
+	if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc)
+		return;
+
+	/* free all mbufs that are valid in the ring */
+	for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask)
+		rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+	rxq->rxrearm_nb = rxq->nb_rx_desc;
+
+	/* set all entries to NULL */
+	memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
+}
+
+static inline int
+i40e_rxq_vec_setup_default(struct i40e_rx_queue *rxq)
+{
+	uintptr_t p;
+	struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
+
+	mb_def.nb_segs = 1;
+	mb_def.data_off = RTE_PKTMBUF_HEADROOM;
+	mb_def.port = rxq->port_id;
+	rte_mbuf_refcnt_set(&mb_def, 1);
+
+	/* prevent compiler reordering: rearm_data covers previous fields */
+	rte_compiler_barrier();
+	p = (uintptr_t)&mb_def.rearm_data;
+	rxq->mbuf_initializer = *(uint64_t *)p;
+	return 0;
+}
+
+static inline int
+i40e_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev *dev)
+{
+#ifndef RTE_LIBRTE_IEEE1588
+	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
+	struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf;
+
+#ifndef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
+	/* whithout rx ol_flags, no VP flag report */
+	if (rxmode->hw_vlan_strip != 0 ||
+	    rxmode->hw_vlan_extend != 0)
+		return -1;
+#endif
+
+	/* no fdir support */
+	if (fconf->mode != RTE_FDIR_MODE_NONE)
+		return -1;
+
+	 /* - no csum error report support
+	 * - no header split support
+	 */
+	if (rxmode->hw_ip_checksum == 1 ||
+	    rxmode->header_split == 1)
+		return -1;
+
+	return 0;
+#else
+	RTE_SET_USED(dev);
+	return -1;
+#endif
+}
+#endif
-- 
2.4.11

Re: [dpdk-dev] [PATCH 1/5] i40e: extract non-x86 specific code from vector driver

[Zhang, Qi Z]

Hi Jianbo

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jianbo Liu
> Sent: Wednesday, August 24, 2016 5:54 PM
> To: Zhang, Helin <helin.zhang@intel.com>; Wu, Jingjing
> <jingjing.wu@intel.com>; jerin.jacob@caviumnetworks.com; dev@dpdk.org
> Cc: Jianbo Liu <jianbo.liu@linaro.org>
> Subject: [dpdk-dev] [PATCH 1/5] i40e: extract non-x86 specific code from vector
> driver
> 
> move scalar code which does not use x86 intrinsic functions to new file
> "i40e_rxtx_vec_common.h", while keeping x86 code in i40e_rxtx_vec.c.
> This allows the scalar code to to be shared among vector drivers for different
> platforms.
> 
> Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
> ---
>  drivers/net/i40e/i40e_rxtx_vec.c        | 184 +-----------------------
>  drivers/net/i40e/i40e_rxtx_vec_common.h | 239
> ++++++++++++++++++++++++++++++++
>  2 files changed, 243 insertions(+), 180 deletions(-)  create mode 100644
> drivers/net/i40e/i40e_rxtx_vec_common.h
> 
> diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec.c
> index 51fb282..f847469 100644
> --- a/drivers/net/i40e/i40e_rxtx_vec.c
> +++ b/drivers/net/i40e/i40e_rxtx_vec.c
> @@ -39,6 +39,7 @@
>  #include "base/i40e_type.h"
>  #include "i40e_ethdev.h"
>  #include "i40e_rxtx.h"
> +#include "i40e_rxtx_vec_common.h"
> 
>  #include <tmmintrin.h>
> 
> @@ -421,68 +422,6 @@ i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf
> **rx_pkts,
>  	return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);  }
> 
> -static inline uint16_t
> -reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
> -		   uint16_t nb_bufs, uint8_t *split_flags)
> -{
> -	struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/
> -	struct rte_mbuf *start = rxq->pkt_first_seg;
> -	struct rte_mbuf *end =  rxq->pkt_last_seg;
> -	unsigned pkt_idx, buf_idx;
> -
> -	for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
> -		if (end != NULL) {
> -			/* processing a split packet */
> -			end->next = rx_bufs[buf_idx];
> -			rx_bufs[buf_idx]->data_len += rxq->crc_len;
> -
> -			start->nb_segs++;
> -			start->pkt_len += rx_bufs[buf_idx]->data_len;
> -			end = end->next;
> -
> -			if (!split_flags[buf_idx]) {
> -				/* it's the last packet of the set */
> -				start->hash = end->hash;
> -				start->ol_flags = end->ol_flags;
> -				/* we need to strip crc for the whole packet */
> -				start->pkt_len -= rxq->crc_len;
> -				if (end->data_len > rxq->crc_len) {
> -					end->data_len -= rxq->crc_len;
> -				} else {
> -					/* free up last mbuf */
> -					struct rte_mbuf *secondlast = start;
> -
> -					while (secondlast->next != end)
> -						secondlast = secondlast->next;
> -					secondlast->data_len -= (rxq->crc_len -
> -							end->data_len);
> -					secondlast->next = NULL;
> -					rte_pktmbuf_free_seg(end);
> -					end = secondlast;
> -				}
> -				pkts[pkt_idx++] = start;
> -				start = end = NULL;
> -			}
> -		} else {
> -			/* not processing a split packet */
> -			if (!split_flags[buf_idx]) {
> -				/* not a split packet, save and skip */
> -				pkts[pkt_idx++] = rx_bufs[buf_idx];
> -				continue;
> -			}
> -			end = start = rx_bufs[buf_idx];
> -			rx_bufs[buf_idx]->data_len += rxq->crc_len;
> -			rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
> -		}
> -	}
> -
> -	/* save the partial packet for next time */
> -	rxq->pkt_first_seg = start;
> -	rxq->pkt_last_seg = end;
> -	memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
> -	return pkt_idx;
> -}
> -
>   /* vPMD receive routine that reassembles scattered packets
>   * Notice:
>   * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet @@
> -548,73 +487,6 @@ vtx(volatile struct i40e_tx_desc *txdp,
>  		vtx1(txdp, *pkt, flags);
>  }
> 
> -static inline int __attribute__((always_inline)) -i40e_tx_free_bufs(struct
> i40e_tx_queue *txq) -{
> -	struct i40e_tx_entry *txep;
> -	uint32_t n;
> -	uint32_t i;
> -	int nb_free = 0;
> -	struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
> -
> -	/* check DD bits on threshold descriptor */
> -	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
> -			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
> -			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
> -		return 0;
> -
> -	n = txq->tx_rs_thresh;
> -
> -	 /* first buffer to free from S/W ring is at index
> -	  * tx_next_dd - (tx_rs_thresh-1)
> -	  */
> -	txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
> -	m = __rte_pktmbuf_prefree_seg(txep[0].mbuf);
> -	if (likely(m != NULL)) {
> -		free[0] = m;
> -		nb_free = 1;
> -		for (i = 1; i < n; i++) {
> -			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
> -			if (likely(m != NULL)) {
> -				if (likely(m->pool == free[0]->pool)) {
> -					free[nb_free++] = m;
> -				} else {
> -					rte_mempool_put_bulk(free[0]->pool,
> -							     (void *)free,
> -							     nb_free);
> -					free[0] = m;
> -					nb_free = 1;
> -				}
> -			}
> -		}
> -		rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
> -	} else {
> -		for (i = 1; i < n; i++) {
> -			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
> -			if (m != NULL)
> -				rte_mempool_put(m->pool, m);
> -		}
> -	}
> -
> -	/* buffers were freed, update counters */
> -	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
> -	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
> -	if (txq->tx_next_dd >= txq->nb_tx_desc)
> -		txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
> -
> -	return txq->tx_rs_thresh;
> -}
> -
> -static inline void __attribute__((always_inline)) -tx_backlog_entry(struct
> i40e_tx_entry *txep,
> -		 struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
> -{
> -	int i;
> -
> -	for (i = 0; i < (int)nb_pkts; ++i)
> -		txep[i].mbuf = tx_pkts[i];
> -}
> -
>  uint16_t
>  i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
>  		   uint16_t nb_pkts)
> @@ -685,37 +557,13 @@ i40e_xmit_pkts_vec(void *tx_queue, struct
> rte_mbuf **tx_pkts,  void __attribute__((cold))
> i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)  {
> -	const unsigned mask = rxq->nb_rx_desc - 1;
> -	unsigned i;
> -
> -	if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc)
> -		return;
> -
> -	/* free all mbufs that are valid in the ring */
> -	for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask)
> -		rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
> -	rxq->rxrearm_nb = rxq->nb_rx_desc;
> -
> -	/* set all entries to NULL */
> -	memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
> +	_i40e_rx_queue_release_mbufs_vec(rxq);
>  }
> 
>  int __attribute__((cold))
>  i40e_rxq_vec_setup(struct i40e_rx_queue *rxq)  {
> -	uintptr_t p;
> -	struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
> -
> -	mb_def.nb_segs = 1;
> -	mb_def.data_off = RTE_PKTMBUF_HEADROOM;
> -	mb_def.port = rxq->port_id;
> -	rte_mbuf_refcnt_set(&mb_def, 1);
> -
> -	/* prevent compiler reordering: rearm_data covers previous fields */
> -	rte_compiler_barrier();
> -	p = (uintptr_t)&mb_def.rearm_data;
> -	rxq->mbuf_initializer = *(uint64_t *)p;
> -	return 0;
> +	return i40e_rxq_vec_setup_default(rxq);
>  }
> 
>  int __attribute__((cold))
> @@ -728,34 +576,10 @@ int __attribute__((cold))
> i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)  {  #ifndef
> RTE_LIBRTE_IEEE1588
> -	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
> -	struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf;
> -
>  	/* need SSE4.1 support */
>  	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
>  		return -1;
> -
> -#ifndef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
> -	/* whithout rx ol_flags, no VP flag report */
> -	if (rxmode->hw_vlan_strip != 0 ||
> -	    rxmode->hw_vlan_extend != 0)
> -		return -1;
>  #endif
> 
> -	/* no fdir support */
> -	if (fconf->mode != RTE_FDIR_MODE_NONE)
> -		return -1;
> -
> -	 /* - no csum error report support
> -	 * - no header split support
> -	 */
> -	if (rxmode->hw_ip_checksum == 1 ||
> -	    rxmode->header_split == 1)
> -		return -1;
> -
> -	return 0;
> -#else
> -	RTE_SET_USED(dev);
> -	return -1;
> -#endif
> +	return i40e_rx_vec_dev_conf_condition_check_default(dev);
>  }
> diff --git a/drivers/net/i40e/i40e_rxtx_vec_common.h
> b/drivers/net/i40e/i40e_rxtx_vec_common.h
> new file mode 100644
> index 0000000..b31b39e
> --- /dev/null
> +++ b/drivers/net/i40e/i40e_rxtx_vec_common.h
> @@ -0,0 +1,239 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
> NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
> FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
> INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
> BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
> LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
> AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
> TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
> OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> DAMAGE.
> + */
> +
> +#ifndef _I40E_RXTX_VEC_COMMON_H_
> +#define _I40E_RXTX_VEC_COMMON_H_
> +#include <stdint.h>
> +#include <rte_ethdev.h>
> +#include <rte_malloc.h>
> +
> +#include "i40e_ethdev.h"
> +#include "i40e_rxtx.h"
> +
> +static inline uint16_t
> +reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
> +		   uint16_t nb_bufs, uint8_t *split_flags) {
> +	struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/
> +	struct rte_mbuf *start = rxq->pkt_first_seg;
> +	struct rte_mbuf *end =  rxq->pkt_last_seg;
> +	unsigned pkt_idx, buf_idx;
> +
> +	for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
> +		if (end != NULL) {
> +			/* processing a split packet */
> +			end->next = rx_bufs[buf_idx];
> +			rx_bufs[buf_idx]->data_len += rxq->crc_len;
> +
> +			start->nb_segs++;
> +			start->pkt_len += rx_bufs[buf_idx]->data_len;
> +			end = end->next;
> +
> +			if (!split_flags[buf_idx]) {
> +				/* it's the last packet of the set */
> +				start->hash = end->hash;
> +				start->ol_flags = end->ol_flags;
> +				/* we need to strip crc for the whole packet */
> +				start->pkt_len -= rxq->crc_len;
> +				if (end->data_len > rxq->crc_len) {
> +					end->data_len -= rxq->crc_len;
> +				} else {
> +					/* free up last mbuf */
> +					struct rte_mbuf *secondlast = start;
> +
> +					while (secondlast->next != end)
> +						secondlast = secondlast->next;
> +					secondlast->data_len -= (rxq->crc_len -
> +							end->data_len);
> +					secondlast->next = NULL;
> +					rte_pktmbuf_free_seg(end);
> +					end = secondlast;
> +				}
> +				pkts[pkt_idx++] = start;
> +				start = end = NULL;
> +			}
> +		} else {
> +			/* not processing a split packet */
> +			if (!split_flags[buf_idx]) {
> +				/* not a split packet, save and skip */
> +				pkts[pkt_idx++] = rx_bufs[buf_idx];
> +				continue;
> +			}
> +			end = start = rx_bufs[buf_idx];
> +			rx_bufs[buf_idx]->data_len += rxq->crc_len;
> +			rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
> +		}
> +	}
> +
> +	/* save the partial packet for next time */
> +	rxq->pkt_first_seg = start;
> +	rxq->pkt_last_seg = end;
> +	memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
> +	return pkt_idx;
> +}
> +
> +static inline int __attribute__((always_inline))
> +i40e_tx_free_bufs(struct i40e_tx_queue *txq) {
> +	struct i40e_tx_entry *txep;
> +	uint32_t n;
> +	uint32_t i;
> +	int nb_free = 0;
> +	struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
> +
> +	/* check DD bits on threshold descriptor */
> +	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
> +			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
> +			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
> +		return 0;
> +
> +	n = txq->tx_rs_thresh;
> +
> +	 /* first buffer to free from S/W ring is at index
> +	  * tx_next_dd - (tx_rs_thresh-1)
> +	  */
> +	txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
> +	m = __rte_pktmbuf_prefree_seg(txep[0].mbuf);
> +	if (likely(m != NULL)) {
> +		free[0] = m;
> +		nb_free = 1;
> +		for (i = 1; i < n; i++) {
> +			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
> +			if (likely(m != NULL)) {
> +				if (likely(m->pool == free[0]->pool)) {
> +					free[nb_free++] = m;
> +				} else {
> +					rte_mempool_put_bulk(free[0]->pool,
> +							     (void *)free,
> +							     nb_free);
> +					free[0] = m;
> +					nb_free = 1;
> +				}
> +			}
> +		}
> +		rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
> +	} else {
> +		for (i = 1; i < n; i++) {
> +			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
> +			if (m != NULL)
> +				rte_mempool_put(m->pool, m);
> +		}
> +	}
> +
> +	/* buffers were freed, update counters */
> +	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
> +	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
> +	if (txq->tx_next_dd >= txq->nb_tx_desc)
> +		txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
> +
> +	return txq->tx_rs_thresh;
> +}
> +
> +static inline void __attribute__((always_inline))
> +tx_backlog_entry(struct i40e_tx_entry *txep,
> +		 struct rte_mbuf **tx_pkts, uint16_t nb_pkts) {
> +	int i;
> +
> +	for (i = 0; i < (int)nb_pkts; ++i)
> +		txep[i].mbuf = tx_pkts[i];
> +}
> +
> +static inline void
> +_i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) {
> +	const unsigned mask = rxq->nb_rx_desc - 1;
> +	unsigned i;
> +
> +	if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc)
> +		return;
> +
> +	/* free all mbufs that are valid in the ring */
> +	for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask)
> +		rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
> +	rxq->rxrearm_nb = rxq->nb_rx_desc;
> +
> +	/* set all entries to NULL */
> +	memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); }
> +
> +static inline int
> +i40e_rxq_vec_setup_default(struct i40e_rx_queue *rxq) {
> +	uintptr_t p;
> +	struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
> +
> +	mb_def.nb_segs = 1;
> +	mb_def.data_off = RTE_PKTMBUF_HEADROOM;
> +	mb_def.port = rxq->port_id;
> +	rte_mbuf_refcnt_set(&mb_def, 1);
> +
> +	/* prevent compiler reordering: rearm_data covers previous fields */
> +	rte_compiler_barrier();
> +	p = (uintptr_t)&mb_def.rearm_data;
> +	rxq->mbuf_initializer = *(uint64_t *)p;
> +	return 0;
> +}
> +
> +static inline int
> +i40e_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev *dev) {
> +#ifndef RTE_LIBRTE_IEEE1588
> +	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
> +	struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf;
> +
> +#ifndef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
> +	/* whithout rx ol_flags, no VP flag report */
> +	if (rxmode->hw_vlan_strip != 0 ||
> +	    rxmode->hw_vlan_extend != 0)
> +		return -1;
> +#endif
> +
> +	/* no fdir support */
> +	if (fconf->mode != RTE_FDIR_MODE_NONE)
> +		return -1;
> +
> +	 /* - no csum error report support
> +	 * - no header split support
> +	 */
> +	if (rxmode->hw_ip_checksum == 1 ||
> +	    rxmode->header_split == 1)
> +		return -1;
> +
> +	return 0;
> +#else
> +	RTE_SET_USED(dev);
> +	return -1;
> +#endif
> +}
> +#endif
> --
> 2.4.11

Should we rename the function "_40e_rx_queue_release_mbufs_vec" to
"i40e_rx_queue_release_mbufs_vec_default", so functions be wrapped can follow a consistent rule?  
Thanks!
Qi

Re: [dpdk-dev] [PATCH 1/5] i40e: extract non-x86 specific code from vector driver

[Jianbo Liu]

On 12 October 2016 at 10:55, Zhang, Qi Z <qi.z.zhang@intel.com> wrote:
> Hi Jianbo
>
>> -----Original Message-----
>> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jianbo Liu
>> Sent: Wednesday, August 24, 2016 5:54 PM
>> To: Zhang, Helin <helin.zhang@intel.com>; Wu, Jingjing
>> <jingjing.wu@intel.com>; jerin.jacob@caviumnetworks.com; dev@dpdk.org
>> Cc: Jianbo Liu <jianbo.liu@linaro.org>
>> Subject: [dpdk-dev] [PATCH 1/5] i40e: extract non-x86 specific code from vector
>> driver
>>
>> move scalar code which does not use x86 intrinsic functions to new file
>> "i40e_rxtx_vec_common.h", while keeping x86 code in i40e_rxtx_vec.c.
>> This allows the scalar code to to be shared among vector drivers for different
>> platforms.
>>
>> Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
>> ---
...
>
> Should we rename the function "_40e_rx_queue_release_mbufs_vec" to
> "i40e_rx_queue_release_mbufs_vec_default", so functions be wrapped can follow a consistent rule?

I think these two ways are different.
For func/_func, _func implements what func needs to do, they are same.
We needs _func inline, to be called in different ARCHs.
But for func/func_default, func_default is the default behavior, but
you can use or not-use it in func.

Re: [dpdk-dev] [PATCH 1/5] i40e: extract non-x86 specific code from vector driver

[Zhang, Qi Z]

Hi Jianbo

> -----Original Message-----
> From: Jianbo Liu [mailto:jianbo.liu@linaro.org]
> Sent: Thursday, October 13, 2016 11:00 AM
> To: Zhang, Qi Z <qi.z.zhang@intel.com>
> Cc: Zhang, Helin <helin.zhang@intel.com>; Wu, Jingjing
> <jingjing.wu@intel.com>; jerin.jacob@caviumnetworks.com; dev@dpdk.org
> Subject: Re: [dpdk-dev] [PATCH 1/5] i40e: extract non-x86 specific code from
> vector driver
> 
> On 12 October 2016 at 10:55, Zhang, Qi Z <qi.z.zhang@intel.com> wrote:
> > Hi Jianbo
> >
> >> -----Original Message-----
> >> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jianbo Liu
> >> Sent: Wednesday, August 24, 2016 5:54 PM
> >> To: Zhang, Helin <helin.zhang@intel.com>; Wu, Jingjing
> >> <jingjing.wu@intel.com>; jerin.jacob@caviumnetworks.com;
> dev@dpdk.org
> >> Cc: Jianbo Liu <jianbo.liu@linaro.org>
> >> Subject: [dpdk-dev] [PATCH 1/5] i40e: extract non-x86 specific code
> >> from vector driver
> >>
> >> move scalar code which does not use x86 intrinsic functions to new
> >> file "i40e_rxtx_vec_common.h", while keeping x86 code in
> i40e_rxtx_vec.c.
> >> This allows the scalar code to to be shared among vector drivers for
> >> different platforms.
> >>
> >> Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
> >> ---
> ...
> >
> > Should we rename the function "_40e_rx_queue_release_mbufs_vec" to
> > "i40e_rx_queue_release_mbufs_vec_default", so functions be wrapped
> can follow a consistent rule?
> 
> I think these two ways are different.
> For func/_func, _func implements what func needs to do, they are same.
> We needs _func inline, to be called in different ARCHs.
> But for func/func_default, func_default is the default behavior, but you can
> use or not-use it in func.

Got your point, I also saw ixgbe is implemented in the same way.
Thanks!
Qi

[dpdk-dev] [PATCH 2/5] i40e: implement vector PMD for ARM architecture

[Jianbo Liu]

Use ARM NEON intrinsic to implement i40e vPMD

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 drivers/net/i40e/Makefile             |   4 +
 drivers/net/i40e/i40e_rxtx_vec_neon.c | 581 ++++++++++++++++++++++++++++++++++
 2 files changed, 585 insertions(+)
 create mode 100644 drivers/net/i40e/i40e_rxtx_vec_neon.c

diff --git a/drivers/net/i40e/Makefile b/drivers/net/i40e/Makefile
index 53fe145..9e92b38 100644
--- a/drivers/net/i40e/Makefile
+++ b/drivers/net/i40e/Makefile
@@ -97,7 +97,11 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_dcb.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_rxtx.c
+ifeq ($(CONFIG_RTE_ARCH_ARM64),y)
+SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_neon.c
+else
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec.c
+endif
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev_vf.c
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_pf.c
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_fdir.c
diff --git a/drivers/net/i40e/i40e_rxtx_vec_neon.c b/drivers/net/i40e/i40e_rxtx_vec_neon.c
new file mode 100644
index 0000000..015fa9f
--- /dev/null
+++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c
@@ -0,0 +1,581 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2016, Linaro Limited
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+
+#include "base/i40e_prototype.h"
+#include "base/i40e_type.h"
+#include "i40e_ethdev.h"
+#include "i40e_rxtx.h"
+#include "i40e_rxtx_vec_common.h"
+
+#include <arm_neon.h>
+
+#pragma GCC diagnostic ignored "-Wcast-qual"
+
+static inline void
+i40e_rxq_rearm(struct i40e_rx_queue *rxq)
+{
+	int i;
+	uint16_t rx_id;
+	volatile union i40e_rx_desc *rxdp;
+	struct i40e_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
+	struct rte_mbuf *mb0, *mb1;
+	uint64x2_t dma_addr0, dma_addr1;
+	uint64x2_t zero = vdupq_n_u64(0);
+	uint64_t paddr;
+	uint8x8_t p;
+
+	rxdp = rxq->rx_ring + rxq->rxrearm_start;
+
+	/* Pull 'n' more MBUFs into the software ring */
+	if (unlikely(rte_mempool_get_bulk(rxq->mp,
+					  (void *)rxep,
+					  RTE_I40E_RXQ_REARM_THRESH) < 0)) {
+		if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >=
+		    rxq->nb_rx_desc) {
+			for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) {
+				rxep[i].mbuf = &rxq->fake_mbuf;
+				vst1q_u64((uint64_t *)&rxdp[i].read, zero);
+			}
+		}
+		rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
+			RTE_I40E_RXQ_REARM_THRESH;
+		return;
+	}
+
+	p = vld1_u8((uint8_t *)&rxq->mbuf_initializer);
+
+	/* Initialize the mbufs in vector, process 2 mbufs in one loop */
+	for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) {
+		mb0 = rxep[0].mbuf;
+		mb1 = rxep[1].mbuf;
+
+		 /* Flush mbuf with pkt template.
+		 * Data to be rearmed is 6 bytes long.
+		 * Though, RX will overwrite ol_flags that are coming next
+		 * anyway. So overwrite whole 8 bytes with one load:
+		 * 6 bytes of rearm_data plus first 2 bytes of ol_flags.
+		 */
+		vst1_u8((uint8_t *)&mb0->rearm_data, p);
+		paddr = mb0->buf_physaddr + RTE_PKTMBUF_HEADROOM;
+		dma_addr0 = vdupq_n_u64(paddr);
+
+		/* flush desc with pa dma_addr */
+		vst1q_u64((uint64_t *)&rxdp++->read, dma_addr0);
+
+		vst1_u8((uint8_t *)&mb1->rearm_data, p);
+		paddr = mb1->buf_physaddr + RTE_PKTMBUF_HEADROOM;
+		dma_addr1 = vdupq_n_u64(paddr);
+		vst1q_u64((uint64_t *)&rxdp++->read, dma_addr1);
+	}
+
+	rxq->rxrearm_start += RTE_I40E_RXQ_REARM_THRESH;
+	if (rxq->rxrearm_start >= rxq->nb_rx_desc)
+		rxq->rxrearm_start = 0;
+
+	rxq->rxrearm_nb -= RTE_I40E_RXQ_REARM_THRESH;
+
+	rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
+			     (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
+
+	/* Update the tail pointer on the NIC */
+	I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+}
+
+/* Handling the offload flags (olflags) field takes computation
+ * time when receiving packets. Therefore we provide a flag to disable
+ * the processing of the olflags field when they are not needed. This
+ * gives improved performance, at the cost of losing the offload info
+ * in the received packet
+ */
+#ifdef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
+
+static inline void
+desc_to_olflags_v(uint16x8_t staterr, struct rte_mbuf **rx_pkts)
+{
+	uint16x8_t vlan0, vlan1, rss;
+	union {
+		uint16_t e[4];
+		uint64_t dword;
+	} vol;
+
+	/* mask everything except RSS, flow director and VLAN flags
+	 * bit2 is for VLAN tag, bit11 for flow director indication
+	 * bit13:12 for RSS indication.
+	 */
+	const uint16x8_t rss_vlan_msk = {
+			0x3804, 0x3804, 0x3804, 0x3804,
+			0x0000, 0x0000, 0x0000, 0x0000};
+
+	/* map rss and vlan type to rss hash and vlan flag */
+	const uint8x16_t vlan_flags = {
+			0, 0, 0, 0,
+			PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED, 0, 0, 0,
+			0, 0, 0, 0,
+			0, 0, 0, 0};
+
+	const uint8x16_t rss_flags = {
+			0, PKT_RX_FDIR, 0, 0,
+			0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR,
+			0, 0, 0, 0,
+			0, 0, 0, 0};
+
+	vlan1 = vandq_u16(staterr, rss_vlan_msk);
+	vlan0 = vreinterpretq_u16_u8(vqtbl1q_u8(vlan_flags,
+						vreinterpretq_u8_u16(vlan1)));
+
+	rss = vshrq_n_u16(vlan1, 11);
+	rss = vreinterpretq_u16_u8(vqtbl1q_u8(rss_flags,
+					      vreinterpretq_u8_u16(rss)));
+
+	vlan0 = vorrq_u16(vlan0, rss);
+	vol.dword = vgetq_lane_u64(vreinterpretq_u64_u16(vlan0), 0);
+
+	rx_pkts[0]->ol_flags = vol.e[0];
+	rx_pkts[1]->ol_flags = vol.e[1];
+	rx_pkts[2]->ol_flags = vol.e[2];
+	rx_pkts[3]->ol_flags = vol.e[3];
+}
+#else
+#define desc_to_olflags_v(staterr, rx_pkts) do {} while (0)
+#endif
+
+#define PKTLEN_SHIFT     10
+
+#define I40E_VPMD_DESC_DD_MASK	0x0001000100010001ULL
+
+ /*
+ * Notice:
+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
+ *   numbers of DD bits
+ */
+static inline uint16_t
+_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+		   uint16_t nb_pkts, uint8_t *split_packet)
+{
+	volatile union i40e_rx_desc *rxdp;
+	struct i40e_rx_entry *sw_ring;
+	uint16_t nb_pkts_recd;
+	int pos;
+	uint64_t var;
+
+	/* mask to shuffle from desc. to mbuf */
+	uint8x16_t shuf_msk = {
+		0xFF, 0xFF,   /* pkt_type set as unknown */
+		0xFF, 0xFF,   /* pkt_type set as unknown */
+		14, 15,       /* octet 15~14, low 16 bits pkt_len */
+		0xFF, 0xFF,   /* skip high 16 bits pkt_len, zero out */
+		14, 15,       /* octet 15~14, 16 bits data_len */
+		2, 3,         /* octet 2~3, low 16 bits vlan_macip */
+		4, 5, 6, 7    /* octet 4~7, 32bits rss */
+		};
+
+	uint8x16_t eop_check = {
+		0x02, 0x00, 0x02, 0x00,
+		0x02, 0x00, 0x02, 0x00,
+		0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00
+		};
+
+	uint16x8_t crc_adjust = {
+		0, 0,         /* ignore pkt_type field */
+		rxq->crc_len, /* sub crc on pkt_len */
+		0,            /* ignore high-16bits of pkt_len */
+		rxq->crc_len, /* sub crc on data_len */
+		0, 0, 0       /* ignore non-length fields */
+		};
+
+	/* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */
+	nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST);
+
+	/* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */
+	nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP);
+
+	/* Just the act of getting into the function from the application is
+	 * going to cost about 7 cycles
+	 */
+	rxdp = rxq->rx_ring + rxq->rx_tail;
+
+	rte_prefetch_non_temporal(rxdp);
+
+	/* See if we need to rearm the RX queue - gives the prefetch a bit
+	 * of time to act
+	 */
+	if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH)
+		i40e_rxq_rearm(rxq);
+
+	/* Before we start moving massive data around, check to see if
+	 * there is actually a packet available
+	 */
+	if (!(rxdp->wb.qword1.status_error_len &
+			rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
+		return 0;
+
+	/* Cache is empty -> need to scan the buffer rings, but first move
+	 * the next 'n' mbufs into the cache
+	 */
+	sw_ring = &rxq->sw_ring[rxq->rx_tail];
+
+	/* A. load 4 packet in one loop
+	 * [A*. mask out 4 unused dirty field in desc]
+	 * B. copy 4 mbuf point from swring to rx_pkts
+	 * C. calc the number of DD bits among the 4 packets
+	 * [C*. extract the end-of-packet bit, if requested]
+	 * D. fill info. from desc to mbuf
+	 */
+
+	for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
+			pos += RTE_I40E_DESCS_PER_LOOP,
+			rxdp += RTE_I40E_DESCS_PER_LOOP) {
+		uint64x2_t descs[RTE_I40E_DESCS_PER_LOOP];
+		uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+		uint16x8x2_t sterr_tmp1, sterr_tmp2;
+		uint64x2_t mbp1, mbp2;
+		uint16x8_t staterr;
+		uint16x8_t tmp;
+		uint64_t stat;
+
+		int32x4_t len_shl = {0, 0, 0, PKTLEN_SHIFT};
+
+		/* B.1 load 1 mbuf point */
+		mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
+		/* Read desc statuses backwards to avoid race condition */
+		/* A.1 load 4 pkts desc */
+		descs[3] =  vld1q_u64((uint64_t *)(rxdp + 3));
+		rte_rmb();
+
+		/* B.2 copy 2 mbuf point into rx_pkts  */
+		vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1);
+
+		/* B.1 load 1 mbuf point */
+		mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
+
+		descs[2] =  vld1q_u64((uint64_t *)(rxdp + 2));
+		/* B.1 load 2 mbuf point */
+		descs[1] =  vld1q_u64((uint64_t *)(rxdp + 1));
+		descs[0] =  vld1q_u64((uint64_t *)(rxdp));
+
+		/* B.2 copy 2 mbuf point into rx_pkts  */
+		vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2);
+
+		if (split_packet) {
+			rte_mbuf_prefetch_part2(rx_pkts[pos]);
+			rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
+			rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
+			rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
+		}
+
+		/* avoid compiler reorder optimization */
+		rte_compiler_barrier();
+
+		/* pkt 3,4 shift the pktlen field to be 16-bit aligned*/
+		uint32x4_t len3 = vshlq_u32(vreinterpretq_u32_u64(descs[3]),
+					    len_shl);
+		descs[3] = vreinterpretq_u64_u32(len3);
+		uint32x4_t len2 = vshlq_u32(vreinterpretq_u32_u64(descs[2]),
+					    len_shl);
+		descs[2] = vreinterpretq_u64_u32(len2);
+
+		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
+		pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk);
+		pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk);
+
+		/* C.1 4=>2 filter staterr info only */
+		sterr_tmp2 = vzipq_u16(vreinterpretq_u16_u64(descs[1]),
+				       vreinterpretq_u16_u64(descs[3]));
+		/* C.1 4=>2 filter staterr info only */
+		sterr_tmp1 = vzipq_u16(vreinterpretq_u16_u64(descs[0]),
+				       vreinterpretq_u16_u64(descs[2]));
+
+		/* C.2 get 4 pkts staterr value  */
+		staterr = vzipq_u16(sterr_tmp1.val[1],
+				    sterr_tmp2.val[1]).val[0];
+		stat = vgetq_lane_u64(vreinterpretq_u64_u16(staterr), 0);
+
+		desc_to_olflags_v(staterr, &rx_pkts[pos]);
+
+		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
+		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
+		pkt_mb4 = vreinterpretq_u8_u16(tmp);
+		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
+		pkt_mb3 = vreinterpretq_u8_u16(tmp);
+
+		/* pkt 1,2 shift the pktlen field to be 16-bit aligned*/
+		uint32x4_t len1 = vshlq_u32(vreinterpretq_u32_u64(descs[1]),
+					    len_shl);
+		descs[1] = vreinterpretq_u64_u32(len1);
+		uint32x4_t len0 = vshlq_u32(vreinterpretq_u32_u64(descs[0]),
+					    len_shl);
+		descs[0] = vreinterpretq_u64_u32(len0);
+
+		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
+		pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), shuf_msk);
+		pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), shuf_msk);
+
+		/* D.3 copy final 3,4 data to rx_pkts */
+		vst1q_u8((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
+				 pkt_mb4);
+		vst1q_u8((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
+				 pkt_mb3);
+
+		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
+		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
+		pkt_mb2 = vreinterpretq_u8_u16(tmp);
+		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
+		pkt_mb1 = vreinterpretq_u8_u16(tmp);
+
+		/* C* extract and record EOP bit */
+		if (split_packet) {
+			uint8x16_t eop_shuf_mask = {
+					0x00, 0x02, 0x04, 0x06,
+					0xFF, 0xFF, 0xFF, 0xFF,
+					0xFF, 0xFF, 0xFF, 0xFF,
+					0xFF, 0xFF, 0xFF, 0xFF};
+			uint8x16_t eop_bits;
+
+			/* and with mask to extract bits, flipping 1-0 */
+			eop_bits = vmvnq_u8(vreinterpretq_u8_u16(staterr));
+			eop_bits = vandq_u8(eop_bits, eop_check);
+			/* the staterr values are not in order, as the count
+			 * count of dd bits doesn't care. However, for end of
+			 * packet tracking, we do care, so shuffle. This also
+			 * compresses the 32-bit values to 8-bit
+			 */
+			eop_bits = vqtbl1q_u8(eop_bits, eop_shuf_mask);
+
+			/* store the resulting 32-bit value */
+			vst1q_lane_u32((uint32_t *)split_packet,
+				       vreinterpretq_u32_u8(eop_bits), 0);
+			split_packet += RTE_I40E_DESCS_PER_LOOP;
+
+			/* zero-out next pointers */
+			rx_pkts[pos]->next = NULL;
+			rx_pkts[pos + 1]->next = NULL;
+			rx_pkts[pos + 2]->next = NULL;
+			rx_pkts[pos + 3]->next = NULL;
+		}
+
+		rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP);
+
+		/* D.3 copy final 1,2 data to rx_pkts */
+		vst1q_u8((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
+			 pkt_mb2);
+		vst1q_u8((void *)&rx_pkts[pos]->rx_descriptor_fields1,
+			 pkt_mb1);
+		/* C.4 calc avaialbe number of desc */
+		var = __builtin_popcountll(stat & I40E_VPMD_DESC_DD_MASK);
+		nb_pkts_recd += var;
+		if (likely(var != RTE_I40E_DESCS_PER_LOOP))
+			break;
+	}
+
+	/* Update our internal tail pointer */
+	rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);
+	rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));
+	rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
+
+	return nb_pkts_recd;
+}
+
+ /*
+ * Notice:
+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
+ *   numbers of DD bits
+ */
+uint16_t
+i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+		   uint16_t nb_pkts)
+{
+	return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
+}
+
+ /* vPMD receive routine that reassembles scattered packets
+ * Notice:
+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
+ *   numbers of DD bits
+ */
+uint16_t
+i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+			     uint16_t nb_pkts)
+{
+
+	struct i40e_rx_queue *rxq = rx_queue;
+	uint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0};
+
+	/* get some new buffers */
+	uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts,
+			split_flags);
+	if (nb_bufs == 0)
+		return 0;
+
+	/* happy day case, full burst + no packets to be joined */
+	const uint64_t *split_fl64 = (uint64_t *)split_flags;
+
+	if (rxq->pkt_first_seg == NULL &&
+			split_fl64[0] == 0 && split_fl64[1] == 0 &&
+			split_fl64[2] == 0 && split_fl64[3] == 0)
+		return nb_bufs;
+
+	/* reassemble any packets that need reassembly*/
+	unsigned i = 0;
+
+	if (rxq->pkt_first_seg == NULL) {
+		/* find the first split flag, and only reassemble then*/
+		while (i < nb_bufs && !split_flags[i])
+			i++;
+		if (i == nb_bufs)
+			return nb_bufs;
+	}
+	return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,
+		&split_flags[i]);
+}
+
+static inline void
+vtx1(volatile struct i40e_tx_desc *txdp,
+		struct rte_mbuf *pkt, uint64_t flags)
+{
+	uint64_t high_qw = (I40E_TX_DESC_DTYPE_DATA |
+			((uint64_t)flags  << I40E_TXD_QW1_CMD_SHIFT) |
+			((uint64_t)pkt->data_len << I40E_TXD_QW1_TX_BUF_SZ_SHIFT));
+
+	uint64x2_t descriptor = {pkt->buf_physaddr + pkt->data_off, high_qw};
+	vst1q_u64((uint64_t *)txdp, descriptor);
+}
+
+static inline void
+vtx(volatile struct i40e_tx_desc *txdp,
+		struct rte_mbuf **pkt, uint16_t nb_pkts,  uint64_t flags)
+{
+	int i;
+
+	for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt)
+		vtx1(txdp, *pkt, flags);
+}
+
+uint16_t
+i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+		   uint16_t nb_pkts)
+{
+	struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;
+	volatile struct i40e_tx_desc *txdp;
+	struct i40e_tx_entry *txep;
+	uint16_t n, nb_commit, tx_id;
+	uint64_t flags = I40E_TD_CMD;
+	uint64_t rs = I40E_TX_DESC_CMD_RS | I40E_TD_CMD;
+	int i;
+
+	/* cross rx_thresh boundary is not allowed */
+	nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh);
+
+	if (txq->nb_tx_free < txq->tx_free_thresh)
+		i40e_tx_free_bufs(txq);
+
+	nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
+	if (unlikely(nb_pkts == 0))
+		return 0;
+
+	tx_id = txq->tx_tail;
+	txdp = &txq->tx_ring[tx_id];
+	txep = &txq->sw_ring[tx_id];
+
+	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
+
+	n = (uint16_t)(txq->nb_tx_desc - tx_id);
+	if (nb_commit >= n) {
+		tx_backlog_entry(txep, tx_pkts, n);
+
+		for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp)
+			vtx1(txdp, *tx_pkts, flags);
+
+		vtx1(txdp, *tx_pkts++, rs);
+
+		nb_commit = (uint16_t)(nb_commit - n);
+
+		tx_id = 0;
+		txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
+
+		/* avoid reach the end of ring */
+		txdp = &txq->tx_ring[tx_id];
+		txep = &txq->sw_ring[tx_id];
+	}
+
+	tx_backlog_entry(txep, tx_pkts, nb_commit);
+
+	vtx(txdp, tx_pkts, nb_commit, flags);
+
+	tx_id = (uint16_t)(tx_id + nb_commit);
+	if (tx_id > txq->tx_next_rs) {
+		txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=
+			rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
+						I40E_TXD_QW1_CMD_SHIFT);
+		txq->tx_next_rs =
+			(uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
+	}
+
+	txq->tx_tail = tx_id;
+
+	I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
+
+	return nb_pkts;
+}
+
+void __attribute__((cold))
+i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)
+{
+	_i40e_rx_queue_release_mbufs_vec(rxq);
+}
+
+int __attribute__((cold))
+i40e_rxq_vec_setup(struct i40e_rx_queue *rxq)
+{
+	return i40e_rxq_vec_setup_default(rxq);
+}
+
+int __attribute__((cold))
+i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq)
+{
+	return 0;
+}
+
+int __attribute__((cold))
+i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)
+{
+	return i40e_rx_vec_dev_conf_condition_check_default(dev);
+}
-- 
2.4.11

Re: [dpdk-dev] [PATCH 2/5] i40e: implement vector PMD for ARM architecture

[Thomas Monjalon]

Hi Jianbo (and other developers of vectorized PMDs),

2016-08-24 15:23, Jianbo Liu:
> Use ARM NEON intrinsic to implement i40e vPMD

Have you tried to use the generic SIMD intrinsics?
We could maintain only one vectorized implementation by using
	__attribute__ ((vector_size (n)))
as described in https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html

I don't know the limitations of the vector builtins (support, performance,
endianness, etc) but it is worth making a try.
Currently we target to support each PMD for SSE/AVX, Altivec and NEON.
Is there any volunteers working on Intel, POWER and ARM to try converting
the existing codebase?

Thanks

Re: [dpdk-dev] [PATCH 2/5] i40e: implement vector PMD for ARM architecture

[Zhang, Qi Z]

Hi Jianbo:

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jianbo Liu
> Sent: Wednesday, August 24, 2016 5:54 PM
> To: Zhang, Helin <helin.zhang@intel.com>; Wu, Jingjing
> <jingjing.wu@intel.com>; jerin.jacob@caviumnetworks.com; dev@dpdk.org
> Cc: Jianbo Liu <jianbo.liu@linaro.org>
> Subject: [dpdk-dev] [PATCH 2/5] i40e: implement vector PMD for ARM
> architecture
> 
> Use ARM NEON intrinsic to implement i40e vPMD
> 
> Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
> ---
>  drivers/net/i40e/Makefile             |   4 +
>  drivers/net/i40e/i40e_rxtx_vec_neon.c | 581
> ++++++++++++++++++++++++++++++++++
>  2 files changed, 585 insertions(+)
>  create mode 100644 drivers/net/i40e/i40e_rxtx_vec_neon.c
> 
> diff --git a/drivers/net/i40e/Makefile b/drivers/net/i40e/Makefile index
> 53fe145..9e92b38 100644
> --- a/drivers/net/i40e/Makefile
> +++ b/drivers/net/i40e/Makefile
> @@ -97,7 +97,11 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_dcb.c
> 
>  SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev.c
>  SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_rxtx.c
> +ifeq ($(CONFIG_RTE_ARCH_ARM64),y)
> +SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_neon.c
> else
>  SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec.c
> +endif
>  SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev_vf.c
>  SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_pf.c
>  SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_fdir.c diff --git
> a/drivers/net/i40e/i40e_rxtx_vec_neon.c
> b/drivers/net/i40e/i40e_rxtx_vec_neon.c
> new file mode 100644
> index 0000000..015fa9f
> --- /dev/null
> +++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c
> @@ -0,0 +1,581 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
> + *   Copyright(c) 2016, Linaro Limited
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
> NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
> FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
> INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
> BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
> LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
> AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
> TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
> OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> DAMAGE.
> + */
> +
> +#include <stdint.h>
> +#include <rte_ethdev.h>
> +#include <rte_malloc.h>
> +
> +#include "base/i40e_prototype.h"
> +#include "base/i40e_type.h"
> +#include "i40e_ethdev.h"
> +#include "i40e_rxtx.h"
> +#include "i40e_rxtx_vec_common.h"
> +
> +#include <arm_neon.h>
> +
> +#pragma GCC diagnostic ignored "-Wcast-qual"
> +
> +static inline void
> +i40e_rxq_rearm(struct i40e_rx_queue *rxq) {
> +	int i;
> +	uint16_t rx_id;
> +	volatile union i40e_rx_desc *rxdp;
> +	struct i40e_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
> +	struct rte_mbuf *mb0, *mb1;
> +	uint64x2_t dma_addr0, dma_addr1;
> +	uint64x2_t zero = vdupq_n_u64(0);
> +	uint64_t paddr;
> +	uint8x8_t p;
> +
> +	rxdp = rxq->rx_ring + rxq->rxrearm_start;
> +
> +	/* Pull 'n' more MBUFs into the software ring */
> +	if (unlikely(rte_mempool_get_bulk(rxq->mp,
> +					  (void *)rxep,
> +					  RTE_I40E_RXQ_REARM_THRESH) < 0)) {
> +		if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >=
> +		    rxq->nb_rx_desc) {
> +			for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) {
> +				rxep[i].mbuf = &rxq->fake_mbuf;
> +				vst1q_u64((uint64_t *)&rxdp[i].read, zero);
> +			}
> +		}
> +		rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
> +			RTE_I40E_RXQ_REARM_THRESH;
> +		return;
> +	}
> +
> +	p = vld1_u8((uint8_t *)&rxq->mbuf_initializer);
> +
> +	/* Initialize the mbufs in vector, process 2 mbufs in one loop */
> +	for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) {
> +		mb0 = rxep[0].mbuf;
> +		mb1 = rxep[1].mbuf;
> +
> +		 /* Flush mbuf with pkt template.
> +		 * Data to be rearmed is 6 bytes long.
> +		 * Though, RX will overwrite ol_flags that are coming next
> +		 * anyway. So overwrite whole 8 bytes with one load:
> +		 * 6 bytes of rearm_data plus first 2 bytes of ol_flags.
> +		 */
> +		vst1_u8((uint8_t *)&mb0->rearm_data, p);
> +		paddr = mb0->buf_physaddr + RTE_PKTMBUF_HEADROOM;
> +		dma_addr0 = vdupq_n_u64(paddr);
> +
> +		/* flush desc with pa dma_addr */
> +		vst1q_u64((uint64_t *)&rxdp++->read, dma_addr0);
> +
> +		vst1_u8((uint8_t *)&mb1->rearm_data, p);
> +		paddr = mb1->buf_physaddr + RTE_PKTMBUF_HEADROOM;
> +		dma_addr1 = vdupq_n_u64(paddr);
> +		vst1q_u64((uint64_t *)&rxdp++->read, dma_addr1);
> +	}
> +
> +	rxq->rxrearm_start += RTE_I40E_RXQ_REARM_THRESH;
> +	if (rxq->rxrearm_start >= rxq->nb_rx_desc)
> +		rxq->rxrearm_start = 0;
> +
> +	rxq->rxrearm_nb -= RTE_I40E_RXQ_REARM_THRESH;
> +
> +	rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
> +			     (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
> +
> +	/* Update the tail pointer on the NIC */
> +	I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); }
> +
> +/* Handling the offload flags (olflags) field takes computation
> + * time when receiving packets. Therefore we provide a flag to disable
> + * the processing of the olflags field when they are not needed. This
> + * gives improved performance, at the cost of losing the offload info
> + * in the received packet
> + */
> +#ifdef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
> +
> +static inline void
> +desc_to_olflags_v(uint16x8_t staterr, struct rte_mbuf **rx_pkts) {
> +	uint16x8_t vlan0, vlan1, rss;
> +	union {
> +		uint16_t e[4];
> +		uint64_t dword;
> +	} vol;
> +
> +	/* mask everything except RSS, flow director and VLAN flags
> +	 * bit2 is for VLAN tag, bit11 for flow director indication
> +	 * bit13:12 for RSS indication.
> +	 */
> +	const uint16x8_t rss_vlan_msk = {
> +			0x3804, 0x3804, 0x3804, 0x3804,
> +			0x0000, 0x0000, 0x0000, 0x0000};
> +
> +	/* map rss and vlan type to rss hash and vlan flag */
> +	const uint8x16_t vlan_flags = {
> +			0, 0, 0, 0,
> +			PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED, 0, 0, 0,
> +			0, 0, 0, 0,
> +			0, 0, 0, 0};
> +
> +	const uint8x16_t rss_flags = {
> +			0, PKT_RX_FDIR, 0, 0,
> +			0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR,
> +			0, 0, 0, 0,
> +			0, 0, 0, 0};
> +
> +	vlan1 = vandq_u16(staterr, rss_vlan_msk);
> +	vlan0 = vreinterpretq_u16_u8(vqtbl1q_u8(vlan_flags,
> +						vreinterpretq_u8_u16(vlan1)));
> +
> +	rss = vshrq_n_u16(vlan1, 11);
> +	rss = vreinterpretq_u16_u8(vqtbl1q_u8(rss_flags,
> +					      vreinterpretq_u8_u16(rss)));
> +
> +	vlan0 = vorrq_u16(vlan0, rss);
> +	vol.dword = vgetq_lane_u64(vreinterpretq_u64_u16(vlan0), 0);
> +
> +	rx_pkts[0]->ol_flags = vol.e[0];
> +	rx_pkts[1]->ol_flags = vol.e[1];
> +	rx_pkts[2]->ol_flags = vol.e[2];
> +	rx_pkts[3]->ol_flags = vol.e[3];
> +}
> +#else
> +#define desc_to_olflags_v(staterr, rx_pkts) do {} while (0) #endif
> +
> +#define PKTLEN_SHIFT     10
> +
> +#define I40E_VPMD_DESC_DD_MASK	0x0001000100010001ULL
> +
> + /*
> + * Notice:
> + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
> + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan
> RTE_I40E_VPMD_RX_BURST
> + *   numbers of DD bits
> + */
> +static inline uint16_t
> +_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
> +		   uint16_t nb_pkts, uint8_t *split_packet) {
> +	volatile union i40e_rx_desc *rxdp;
> +	struct i40e_rx_entry *sw_ring;
> +	uint16_t nb_pkts_recd;
> +	int pos;
> +	uint64_t var;
> +
> +	/* mask to shuffle from desc. to mbuf */
> +	uint8x16_t shuf_msk = {
> +		0xFF, 0xFF,   /* pkt_type set as unknown */
> +		0xFF, 0xFF,   /* pkt_type set as unknown */
> +		14, 15,       /* octet 15~14, low 16 bits pkt_len */
> +		0xFF, 0xFF,   /* skip high 16 bits pkt_len, zero out */
> +		14, 15,       /* octet 15~14, 16 bits data_len */
> +		2, 3,         /* octet 2~3, low 16 bits vlan_macip */
> +		4, 5, 6, 7    /* octet 4~7, 32bits rss */
> +		};
> +
> +	uint8x16_t eop_check = {
> +		0x02, 0x00, 0x02, 0x00,
> +		0x02, 0x00, 0x02, 0x00,
> +		0x00, 0x00, 0x00, 0x00,
> +		0x00, 0x00, 0x00, 0x00
> +		};
> +
> +	uint16x8_t crc_adjust = {
> +		0, 0,         /* ignore pkt_type field */
> +		rxq->crc_len, /* sub crc on pkt_len */
> +		0,            /* ignore high-16bits of pkt_len */
> +		rxq->crc_len, /* sub crc on data_len */
> +		0, 0, 0       /* ignore non-length fields */
> +		};
> +
> +	/* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */
> +	nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST);
> +
> +	/* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */
> +	nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP);
> +
> +	/* Just the act of getting into the function from the application is
> +	 * going to cost about 7 cycles
> +	 */
> +	rxdp = rxq->rx_ring + rxq->rx_tail;
> +
> +	rte_prefetch_non_temporal(rxdp);
> +
> +	/* See if we need to rearm the RX queue - gives the prefetch a bit
> +	 * of time to act
> +	 */
> +	if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH)
> +		i40e_rxq_rearm(rxq);
> +
> +	/* Before we start moving massive data around, check to see if
> +	 * there is actually a packet available
> +	 */
> +	if (!(rxdp->wb.qword1.status_error_len &
> +			rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
> +		return 0;
> +
> +	/* Cache is empty -> need to scan the buffer rings, but first move
> +	 * the next 'n' mbufs into the cache
> +	 */
> +	sw_ring = &rxq->sw_ring[rxq->rx_tail];
> +
> +	/* A. load 4 packet in one loop
> +	 * [A*. mask out 4 unused dirty field in desc]
> +	 * B. copy 4 mbuf point from swring to rx_pkts
> +	 * C. calc the number of DD bits among the 4 packets
> +	 * [C*. extract the end-of-packet bit, if requested]
> +	 * D. fill info. from desc to mbuf
> +	 */
> +
> +	for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
> +			pos += RTE_I40E_DESCS_PER_LOOP,
> +			rxdp += RTE_I40E_DESCS_PER_LOOP) {
> +		uint64x2_t descs[RTE_I40E_DESCS_PER_LOOP];
> +		uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
> +		uint16x8x2_t sterr_tmp1, sterr_tmp2;
> +		uint64x2_t mbp1, mbp2;
> +		uint16x8_t staterr;
> +		uint16x8_t tmp;
> +		uint64_t stat;
> +
> +		int32x4_t len_shl = {0, 0, 0, PKTLEN_SHIFT};
> +
> +		/* B.1 load 1 mbuf point */
> +		mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
> +		/* Read desc statuses backwards to avoid race condition */
> +		/* A.1 load 4 pkts desc */
> +		descs[3] =  vld1q_u64((uint64_t *)(rxdp + 3));
> +		rte_rmb();
> +
> +		/* B.2 copy 2 mbuf point into rx_pkts  */
> +		vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1);
> +
> +		/* B.1 load 1 mbuf point */
> +		mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
> +
> +		descs[2] =  vld1q_u64((uint64_t *)(rxdp + 2));
> +		/* B.1 load 2 mbuf point */
> +		descs[1] =  vld1q_u64((uint64_t *)(rxdp + 1));
> +		descs[0] =  vld1q_u64((uint64_t *)(rxdp));
> +
> +		/* B.2 copy 2 mbuf point into rx_pkts  */
> +		vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2);
> +
> +		if (split_packet) {
> +			rte_mbuf_prefetch_part2(rx_pkts[pos]);
> +			rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
> +			rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
> +			rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
> +		}
> +
> +		/* avoid compiler reorder optimization */
> +		rte_compiler_barrier();
> +
> +		/* pkt 3,4 shift the pktlen field to be 16-bit aligned*/
> +		uint32x4_t len3 = vshlq_u32(vreinterpretq_u32_u64(descs[3]),
> +					    len_shl);
> +		descs[3] = vreinterpretq_u64_u32(len3);
> +		uint32x4_t len2 = vshlq_u32(vreinterpretq_u32_u64(descs[2]),
> +					    len_shl);
> +		descs[2] = vreinterpretq_u64_u32(len2);
> +
> +		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
> +		pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk);
> +		pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk);
> +
> +		/* C.1 4=>2 filter staterr info only */
> +		sterr_tmp2 = vzipq_u16(vreinterpretq_u16_u64(descs[1]),
> +				       vreinterpretq_u16_u64(descs[3]));
> +		/* C.1 4=>2 filter staterr info only */
> +		sterr_tmp1 = vzipq_u16(vreinterpretq_u16_u64(descs[0]),
> +				       vreinterpretq_u16_u64(descs[2]));
> +
> +		/* C.2 get 4 pkts staterr value  */
> +		staterr = vzipq_u16(sterr_tmp1.val[1],
> +				    sterr_tmp2.val[1]).val[0];
> +		stat = vgetq_lane_u64(vreinterpretq_u64_u16(staterr), 0);
> +
> +		desc_to_olflags_v(staterr, &rx_pkts[pos]);
> +
> +		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
> +		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
> +		pkt_mb4 = vreinterpretq_u8_u16(tmp);
> +		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
> +		pkt_mb3 = vreinterpretq_u8_u16(tmp);
> +
> +		/* pkt 1,2 shift the pktlen field to be 16-bit aligned*/
> +		uint32x4_t len1 = vshlq_u32(vreinterpretq_u32_u64(descs[1]),
> +					    len_shl);
> +		descs[1] = vreinterpretq_u64_u32(len1);
> +		uint32x4_t len0 = vshlq_u32(vreinterpretq_u32_u64(descs[0]),
> +					    len_shl);
> +		descs[0] = vreinterpretq_u64_u32(len0);
> +
> +		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
> +		pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), shuf_msk);
> +		pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), shuf_msk);
> +
> +		/* D.3 copy final 3,4 data to rx_pkts */
> +		vst1q_u8((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
> +				 pkt_mb4);
> +		vst1q_u8((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
> +				 pkt_mb3);
> +
> +		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
> +		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
> +		pkt_mb2 = vreinterpretq_u8_u16(tmp);
> +		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
> +		pkt_mb1 = vreinterpretq_u8_u16(tmp);
> +
> +		/* C* extract and record EOP bit */
> +		if (split_packet) {
> +			uint8x16_t eop_shuf_mask = {
> +					0x00, 0x02, 0x04, 0x06,
> +					0xFF, 0xFF, 0xFF, 0xFF,
> +					0xFF, 0xFF, 0xFF, 0xFF,
> +					0xFF, 0xFF, 0xFF, 0xFF};
> +			uint8x16_t eop_bits;
> +
> +			/* and with mask to extract bits, flipping 1-0 */
> +			eop_bits = vmvnq_u8(vreinterpretq_u8_u16(staterr));
> +			eop_bits = vandq_u8(eop_bits, eop_check);
> +			/* the staterr values are not in order, as the count
> +			 * count of dd bits doesn't care. However, for end of
> +			 * packet tracking, we do care, so shuffle. This also
> +			 * compresses the 32-bit values to 8-bit
> +			 */
> +			eop_bits = vqtbl1q_u8(eop_bits, eop_shuf_mask);
> +
> +			/* store the resulting 32-bit value */
> +			vst1q_lane_u32((uint32_t *)split_packet,
> +				       vreinterpretq_u32_u8(eop_bits), 0);
> +			split_packet += RTE_I40E_DESCS_PER_LOOP;
> +
> +			/* zero-out next pointers */
> +			rx_pkts[pos]->next = NULL;
> +			rx_pkts[pos + 1]->next = NULL;
> +			rx_pkts[pos + 2]->next = NULL;
> +			rx_pkts[pos + 3]->next = NULL;
> +		}
> +
> +		rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP);
> +
> +		/* D.3 copy final 1,2 data to rx_pkts */
> +		vst1q_u8((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
> +			 pkt_mb2);
> +		vst1q_u8((void *)&rx_pkts[pos]->rx_descriptor_fields1,
> +			 pkt_mb1);
> +		/* C.4 calc avaialbe number of desc */
> +		var = __builtin_popcountll(stat & I40E_VPMD_DESC_DD_MASK);
> +		nb_pkts_recd += var;
> +		if (likely(var != RTE_I40E_DESCS_PER_LOOP))
> +			break;
> +	}
> +
> +	/* Update our internal tail pointer */
> +	rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);
> +	rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));
> +	rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
> +
> +	return nb_pkts_recd;
> +}
> +
> + /*
> + * Notice:
> + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
> + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan
> RTE_I40E_VPMD_RX_BURST
> + *   numbers of DD bits
> + */
> +uint16_t
> +i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
> +		   uint16_t nb_pkts)
> +{
> +	return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL); }
> +
> + /* vPMD receive routine that reassembles scattered packets
> + * Notice:
> + * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
> + * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan
> RTE_I40E_VPMD_RX_BURST
> + *   numbers of DD bits
> + */
> +uint16_t
> +i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
> +			     uint16_t nb_pkts)
> +{
> +
> +	struct i40e_rx_queue *rxq = rx_queue;
> +	uint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0};
> +
> +	/* get some new buffers */
> +	uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts,
> +			split_flags);
> +	if (nb_bufs == 0)
> +		return 0;
> +
> +	/* happy day case, full burst + no packets to be joined */
> +	const uint64_t *split_fl64 = (uint64_t *)split_flags;
> +
> +	if (rxq->pkt_first_seg == NULL &&
> +			split_fl64[0] == 0 && split_fl64[1] == 0 &&
> +			split_fl64[2] == 0 && split_fl64[3] == 0)
> +		return nb_bufs;
> +
> +	/* reassemble any packets that need reassembly*/
> +	unsigned i = 0;
> +
> +	if (rxq->pkt_first_seg == NULL) {
> +		/* find the first split flag, and only reassemble then*/
> +		while (i < nb_bufs && !split_flags[i])
> +			i++;
> +		if (i == nb_bufs)
> +			return nb_bufs;
> +	}
> +	return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,
> +		&split_flags[i]);
> +}
> +
> +static inline void
> +vtx1(volatile struct i40e_tx_desc *txdp,
> +		struct rte_mbuf *pkt, uint64_t flags) {
> +	uint64_t high_qw = (I40E_TX_DESC_DTYPE_DATA |
> +			((uint64_t)flags  << I40E_TXD_QW1_CMD_SHIFT) |
> +			((uint64_t)pkt->data_len <<
> I40E_TXD_QW1_TX_BUF_SZ_SHIFT));
> +
> +	uint64x2_t descriptor = {pkt->buf_physaddr + pkt->data_off, high_qw};
> +	vst1q_u64((uint64_t *)txdp, descriptor); }
> +
> +static inline void
> +vtx(volatile struct i40e_tx_desc *txdp,
> +		struct rte_mbuf **pkt, uint16_t nb_pkts,  uint64_t flags) {
> +	int i;
> +
> +	for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt)
> +		vtx1(txdp, *pkt, flags);
> +}
> +
> +uint16_t
> +i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
> +		   uint16_t nb_pkts)
> +{
> +	struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;
> +	volatile struct i40e_tx_desc *txdp;
> +	struct i40e_tx_entry *txep;
> +	uint16_t n, nb_commit, tx_id;
> +	uint64_t flags = I40E_TD_CMD;
> +	uint64_t rs = I40E_TX_DESC_CMD_RS | I40E_TD_CMD;
> +	int i;
> +
> +	/* cross rx_thresh boundary is not allowed */
> +	nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh);
> +
> +	if (txq->nb_tx_free < txq->tx_free_thresh)
> +		i40e_tx_free_bufs(txq);
> +
> +	nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
> +	if (unlikely(nb_pkts == 0))
> +		return 0;
> +
> +	tx_id = txq->tx_tail;
> +	txdp = &txq->tx_ring[tx_id];
> +	txep = &txq->sw_ring[tx_id];
> +
> +	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
> +
> +	n = (uint16_t)(txq->nb_tx_desc - tx_id);
> +	if (nb_commit >= n) {
> +		tx_backlog_entry(txep, tx_pkts, n);
> +
> +		for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp)
> +			vtx1(txdp, *tx_pkts, flags);
> +
> +		vtx1(txdp, *tx_pkts++, rs);
> +
> +		nb_commit = (uint16_t)(nb_commit - n);
> +
> +		tx_id = 0;
> +		txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
> +
> +		/* avoid reach the end of ring */
> +		txdp = &txq->tx_ring[tx_id];
> +		txep = &txq->sw_ring[tx_id];
> +	}
> +
> +	tx_backlog_entry(txep, tx_pkts, nb_commit);
> +
> +	vtx(txdp, tx_pkts, nb_commit, flags);
> +
> +	tx_id = (uint16_t)(tx_id + nb_commit);
> +	if (tx_id > txq->tx_next_rs) {
> +		txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=
> +			rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
> +						I40E_TXD_QW1_CMD_SHIFT);
> +		txq->tx_next_rs =
> +			(uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
> +	}
> +
> +	txq->tx_tail = tx_id;
> +
> +	I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
> +
> +	return nb_pkts;
> +}
> +
> +void __attribute__((cold))
> +i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq) {
> +	_i40e_rx_queue_release_mbufs_vec(rxq);
> +}
> +
> +int __attribute__((cold))
> +i40e_rxq_vec_setup(struct i40e_rx_queue *rxq) {
> +	return i40e_rxq_vec_setup_default(rxq); }
> +
> +int __attribute__((cold))
> +i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq) {
> +	return 0;
> +}
> +
> +int __attribute__((cold))
> +i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) {
> +	return i40e_rx_vec_dev_conf_condition_check_default(dev);
> +}
> --
> 2.4.11

ptype and bad checksum offload is enabled with below patches 
http://dpdk.org/dev/patchwork/patch/16394
http://dpdk.org/dev/patchwork/patch/16395
You may take a look to see if it's necessary to enable them for ARM also.

Thanks!
Qi

Re: [dpdk-dev] [PATCH 2/5] i40e: implement vector PMD for ARM architecture

[Jianbo Liu]

On 12 October 2016 at 10:46, Zhang, Qi Z <qi.z.zhang@intel.com> wrote:
> Hi Jianbo:
>
>> -----Original Message-----
>> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Jianbo Liu
>> Sent: Wednesday, August 24, 2016 5:54 PM
>> To: Zhang, Helin <helin.zhang@intel.com>; Wu, Jingjing
>> <jingjing.wu@intel.com>; jerin.jacob@caviumnetworks.com; dev@dpdk.org
>> Cc: Jianbo Liu <jianbo.liu@linaro.org>
>> Subject: [dpdk-dev] [PATCH 2/5] i40e: implement vector PMD for ARM
>> architecture
>>
>> Use ARM NEON intrinsic to implement i40e vPMD
>>
>> Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
>> ---
>>  drivers/net/i40e/Makefile             |   4 +
>>  drivers/net/i40e/i40e_rxtx_vec_neon.c | 581
>> ++++++++++++++++++++++++++++++++++
>>  2 files changed, 585 insertions(+)
>>  create mode 100644 drivers/net/i40e/i40e_rxtx_vec_neon.c
......
>
> ptype and bad checksum offload is enabled with below patches
> http://dpdk.org/dev/patchwork/patch/16394
> http://dpdk.org/dev/patchwork/patch/16395
> You may take a look to see if it's necessary to enable them for ARM also.
>

Yes, I'll update in the next version. Thanks!

[dpdk-dev] [PATCH 3/5] i40e: enable i40e vector PMD on ARMv8a platform

[Jianbo Liu]

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 config/defconfig_arm64-armv8a-linuxapp-gcc | 1 -
 doc/guides/nics/features/i40e_vec.ini      | 1 +
 doc/guides/nics/features/i40e_vf_vec.ini   | 1 +
 3 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/defconfig_arm64-armv8a-linuxapp-gcc b/config/defconfig_arm64-armv8a-linuxapp-gcc
index 1a17126..d10e1fd 100644
--- a/config/defconfig_arm64-armv8a-linuxapp-gcc
+++ b/config/defconfig_arm64-armv8a-linuxapp-gcc
@@ -46,6 +46,5 @@ CONFIG_RTE_EAL_IGB_UIO=n
 
 CONFIG_RTE_LIBRTE_IVSHMEM=n
 CONFIG_RTE_LIBRTE_FM10K_PMD=n
-CONFIG_RTE_LIBRTE_I40E_PMD=n
 
 CONFIG_RTE_SCHED_VECTOR=n
diff --git a/doc/guides/nics/features/i40e_vec.ini b/doc/guides/nics/features/i40e_vec.ini
index 0953d84..edd6b71 100644
--- a/doc/guides/nics/features/i40e_vec.ini
+++ b/doc/guides/nics/features/i40e_vec.ini
@@ -37,3 +37,4 @@ Linux UIO            = Y
 Linux VFIO           = Y
 x86-32               = Y
 x86-64               = Y
+ARMv8                = Y
diff --git a/doc/guides/nics/features/i40e_vf_vec.ini b/doc/guides/nics/features/i40e_vf_vec.ini
index 2a44bf6..d6674f7 100644
--- a/doc/guides/nics/features/i40e_vf_vec.ini
+++ b/doc/guides/nics/features/i40e_vf_vec.ini
@@ -26,3 +26,4 @@ Linux UIO            = Y
 Linux VFIO           = Y
 x86-32               = Y
 x86-64               = Y
+ARMv8                = Y
-- 
2.4.11

[dpdk-dev] [PATCH 4/5] i40e: make vector driver filenames consistent

[Jianbo Liu]

To be consistent with the naming for ARM NEON implementation,
i40e_rxtx_vec.c is renamed to i40e_rxtx_vec_sse.c.

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 drivers/net/i40e/Makefile                                 | 4 ++--
 drivers/net/i40e/{i40e_rxtx_vec.c => i40e_rxtx_vec_sse.c} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename drivers/net/i40e/{i40e_rxtx_vec.c => i40e_rxtx_vec_sse.c} (100%)

diff --git a/drivers/net/i40e/Makefile b/drivers/net/i40e/Makefile
index 9e92b38..13085fb 100644
--- a/drivers/net/i40e/Makefile
+++ b/drivers/net/i40e/Makefile
@@ -100,7 +100,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_rxtx.c
 ifeq ($(CONFIG_RTE_ARCH_ARM64),y)
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_neon.c
 else
-SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec.c
+SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_sse.c
 endif
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev_vf.c
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_pf.c
@@ -108,7 +108,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_fdir.c
 
 # vector PMD driver needs SSE4.1 support
 ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),)
-CFLAGS_i40e_rxtx_vec.o += -msse4.1
+CFLAGS_i40e_rxtx_vec_sse.o += -msse4.1
 endif
 
 
diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec_sse.c
similarity index 100%
rename from drivers/net/i40e/i40e_rxtx_vec.c
rename to drivers/net/i40e/i40e_rxtx_vec_sse.c
-- 
2.4.11

[dpdk-dev] [PATCH 5/5] maintainers: claim i40e vector PMD on ARM

[Jianbo Liu]

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 MAINTAINERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 6536c6b..5d6ecba 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -150,6 +150,7 @@ F: lib/librte_acl/acl_run_neon.*
 F: lib/librte_lpm/rte_lpm_neon.h
 F: lib/librte_hash/rte*_arm64.h
 F: drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c
+F: drivers/net/i40e/i40e_rxtx_vec_neon.c
 
 EZchip TILE-Gx
 M: Zhigang Lu <zlu@ezchip.com>
-- 
2.4.11

Re: [dpdk-dev] [PATCH 0/5] i40e: vector poll-mode driver on ARM64

[Thomas Monjalon]

2016-08-24 15:23, Jianbo Liu:
> This patch set is to implement i40e vector PMD on ARM64.

Thanks for extending ARM support.

The current NIC support status is:
% git grep -l 'ARM.*=.*Y' doc/guides/nics/features/
doc/guides/nics/features/ixgbe.ini
doc/guides/nics/features/ixgbe_vec.ini
doc/guides/nics/features/ixgbe_vf.ini
doc/guides/nics/features/ixgbe_vf_vec.ini
doc/guides/nics/features/pcap.ini
doc/guides/nics/features/thunderx.ini
doc/guides/nics/features/virtio.ini
doc/guides/nics/features/virtio_vec.ini

To sum it up, only virtio, ixgbe (and specific thunderx device)
seems to be supported on ARM in DPDK 16.07.
Now you are bringing support of i40e on ARM.
Do you plan to support more devices in near future?
Who is interested to do and/or validate ARM support of other drivers?

Re: [dpdk-dev] [PATCH 0/5] i40e: vector poll-mode driver on ARM64

[Shreyansh Jain]

Hi Thomas,

On Wednesday 24 August 2016 04:19 PM, Thomas Monjalon wrote:
> 2016-08-24 15:23, Jianbo Liu:
>> This patch set is to implement i40e vector PMD on ARM64.
>
> Thanks for extending ARM support.
>
> The current NIC support status is:
> % git grep -l 'ARM.*=.*Y' doc/guides/nics/features/
> doc/guides/nics/features/ixgbe.ini
> doc/guides/nics/features/ixgbe_vec.ini
> doc/guides/nics/features/ixgbe_vf.ini
> doc/guides/nics/features/ixgbe_vf_vec.ini
> doc/guides/nics/features/pcap.ini
> doc/guides/nics/features/thunderx.ini
> doc/guides/nics/features/virtio.ini
> doc/guides/nics/features/virtio_vec.ini
>
> To sum it up, only virtio, ixgbe (and specific thunderx device)
> seems to be supported on ARM in DPDK 16.07.
> Now you are bringing support of i40e on ARM.
> Do you plan to support more devices in near future?
> Who is interested to do and/or validate ARM support of other drivers?

We at NXP are currently working on integrating DPDK over NXP's ARM SoC. 
As the implementation is dependent on rte_driver/device [1] and SoC 
framework [2], my plan is to push base infra out in this week and NXP 
RFC early next week.

Other than above, we are also working on supporting ARM Crypto-extension 
based Cryto drivers.

[1] http://dpdk.org/ml/archives/dev/2016-January/031390.html (and 
multiple other patches after this)
[2] http://dpdk.org/ml/archives/dev/2016-May/038486.html

>
>

-
Shreyansh

Re: [dpdk-dev] [PATCH 0/5] i40e: vector poll-mode driver on ARM64

[Jianbo Liu]

On 24 August 2016 at 18:49, Thomas Monjalon <thomas.monjalon@6wind.com> wrote:
> 2016-08-24 15:23, Jianbo Liu:
>> This patch set is to implement i40e vector PMD on ARM64.
>
> Thanks for extending ARM support.
>
> The current NIC support status is:
> % git grep -l 'ARM.*=.*Y' doc/guides/nics/features/
> doc/guides/nics/features/ixgbe.ini
> doc/guides/nics/features/ixgbe_vec.ini
> doc/guides/nics/features/ixgbe_vf.ini
> doc/guides/nics/features/ixgbe_vf_vec.ini
> doc/guides/nics/features/pcap.ini
> doc/guides/nics/features/thunderx.ini
> doc/guides/nics/features/virtio.ini
> doc/guides/nics/features/virtio_vec.ini
>
> To sum it up, only virtio, ixgbe (and specific thunderx device)
> seems to be supported on ARM in DPDK 16.07.
> Now you are bringing support of i40e on ARM.
> Do you plan to support more devices in near future?
> Who is interested to do and/or validate ARM support of other drivers?
>
I'd like to do that as long as I have the hardware :)

Thanks!
Jianbo

Re: [dpdk-dev] [PATCH 0/5] i40e: vector poll-mode driver on ARM64

[Bruce Richardson]

On Wed, Aug 24, 2016 at 03:23:40PM +0530, Jianbo Liu wrote:
> This patch set is to implement i40e vector PMD on ARM64.
> For x86, vPMD is only reorganized, there should be no performance loss.
> 
> Jianbo Liu (5):
>   i40e: extract non-x86 specific code from vector driver
>   i40e: implement vector PMD for ARM architecture
>   i40e: enable i40e vector PMD on ARMv8a platform
>   i40e: make vector driver filenames consistent
>   maintainers: claim i40e vector PMD on ARM
> 
Ping for comment/ack - especially from i40e maintainers.

/Bruce

Re: [dpdk-dev] [PATCH 0/5] i40e: vector poll-mode driver on ARM64

[Thomas Monjalon]

2016-09-19 17:25, Bruce Richardson:
> On Wed, Aug 24, 2016 at 03:23:40PM +0530, Jianbo Liu wrote:
> > This patch set is to implement i40e vector PMD on ARM64.
> > For x86, vPMD is only reorganized, there should be no performance loss.
> > 
> > Jianbo Liu (5):
> >   i40e: extract non-x86 specific code from vector driver
> >   i40e: implement vector PMD for ARM architecture
> >   i40e: enable i40e vector PMD on ARMv8a platform
> >   i40e: make vector driver filenames consistent
> >   maintainers: claim i40e vector PMD on ARM
> > 
> Ping for comment/ack - especially from i40e maintainers.

Why is there no review of these patches for acceleration on ARM?

[dpdk-dev] [PATCH v2 0/5] i40e: vector poll-mode driver on ARM64

[Jianbo Liu]

This patch set is to implement i40e vector PMD on ARM64.
For x86, vPMD is only reorganized, there should be no performance loss.

v1 -> v2
- rebase to dpdk-next-net/rel_16_11

Jianbo Liu (5):
  i40e: extract non-x86 specific code from vector driver
  i40e: implement vector PMD for ARM architecture
  i40e: enable i40e vector PMD on ARMv8a platform
  i40e: make vector driver filenames consistent
  maintainers: claim i40e vector PMD on ARM

 MAINTAINERS                                        |   1 +
 config/defconfig_arm64-armv8a-linuxapp-gcc         |   1 -
 doc/guides/nics/features/i40e_vec.ini              |   1 +
 doc/guides/nics/features/i40e_vf_vec.ini           |   1 +
 drivers/net/i40e/Makefile                          |   8 +-
 drivers/net/i40e/i40e_rxtx_vec_common.h            | 251 +++++++++
 drivers/net/i40e/i40e_rxtx_vec_neon.c              | 614 +++++++++++++++++++++
 .../i40e/{i40e_rxtx_vec.c => i40e_rxtx_vec_sse.c}  | 196 +------
 8 files changed, 878 insertions(+), 195 deletions(-)
 create mode 100644 drivers/net/i40e/i40e_rxtx_vec_common.h
 create mode 100644 drivers/net/i40e/i40e_rxtx_vec_neon.c
 rename drivers/net/i40e/{i40e_rxtx_vec.c => i40e_rxtx_vec_sse.c} (78%)

-- 
2.4.11

[dpdk-dev] [PATCH v2 1/5] i40e: extract non-x86 specific code from vector driver

[Jianbo Liu]

move scalar code which does not use x86 intrinsic functions to new file
"i40e_rxtx_vec_common.h", while keeping x86 code in i40e_rxtx_vec.c.
This allows the scalar code to to be shared among vector drivers for
different platforms.

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 drivers/net/i40e/i40e_rxtx_vec.c        | 196 +------------------------
 drivers/net/i40e/i40e_rxtx_vec_common.h | 251 ++++++++++++++++++++++++++++++++
 2 files changed, 255 insertions(+), 192 deletions(-)
 create mode 100644 drivers/net/i40e/i40e_rxtx_vec_common.h

diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec.c
index 0ee0241..3607312 100644
--- a/drivers/net/i40e/i40e_rxtx_vec.c
+++ b/drivers/net/i40e/i40e_rxtx_vec.c
@@ -39,6 +39,7 @@
 #include "base/i40e_type.h"
 #include "i40e_ethdev.h"
 #include "i40e_rxtx.h"
+#include "i40e_rxtx_vec_common.h"
 
 #include <tmmintrin.h>
 
@@ -445,68 +446,6 @@ i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
 	return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
 }
 
-static inline uint16_t
-reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
-		   uint16_t nb_bufs, uint8_t *split_flags)
-{
-	struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/
-	struct rte_mbuf *start = rxq->pkt_first_seg;
-	struct rte_mbuf *end =  rxq->pkt_last_seg;
-	unsigned pkt_idx, buf_idx;
-
-	for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
-		if (end != NULL) {
-			/* processing a split packet */
-			end->next = rx_bufs[buf_idx];
-			rx_bufs[buf_idx]->data_len += rxq->crc_len;
-
-			start->nb_segs++;
-			start->pkt_len += rx_bufs[buf_idx]->data_len;
-			end = end->next;
-
-			if (!split_flags[buf_idx]) {
-				/* it's the last packet of the set */
-				start->hash = end->hash;
-				start->ol_flags = end->ol_flags;
-				/* we need to strip crc for the whole packet */
-				start->pkt_len -= rxq->crc_len;
-				if (end->data_len > rxq->crc_len) {
-					end->data_len -= rxq->crc_len;
-				} else {
-					/* free up last mbuf */
-					struct rte_mbuf *secondlast = start;
-
-					while (secondlast->next != end)
-						secondlast = secondlast->next;
-					secondlast->data_len -= (rxq->crc_len -
-							end->data_len);
-					secondlast->next = NULL;
-					rte_pktmbuf_free_seg(end);
-					end = secondlast;
-				}
-				pkts[pkt_idx++] = start;
-				start = end = NULL;
-			}
-		} else {
-			/* not processing a split packet */
-			if (!split_flags[buf_idx]) {
-				/* not a split packet, save and skip */
-				pkts[pkt_idx++] = rx_bufs[buf_idx];
-				continue;
-			}
-			end = start = rx_bufs[buf_idx];
-			rx_bufs[buf_idx]->data_len += rxq->crc_len;
-			rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
-		}
-	}
-
-	/* save the partial packet for next time */
-	rxq->pkt_first_seg = start;
-	rxq->pkt_last_seg = end;
-	memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
-	return pkt_idx;
-}
-
  /* vPMD receive routine that reassembles scattered packets
  * Notice:
  * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
@@ -572,73 +511,6 @@ vtx(volatile struct i40e_tx_desc *txdp,
 		vtx1(txdp, *pkt, flags);
 }
 
-static inline int __attribute__((always_inline))
-i40e_tx_free_bufs(struct i40e_tx_queue *txq)
-{
-	struct i40e_tx_entry *txep;
-	uint32_t n;
-	uint32_t i;
-	int nb_free = 0;
-	struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
-
-	/* check DD bits on threshold descriptor */
-	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
-			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
-			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
-		return 0;
-
-	n = txq->tx_rs_thresh;
-
-	 /* first buffer to free from S/W ring is at index
-	  * tx_next_dd - (tx_rs_thresh-1)
-	  */
-	txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
-	m = __rte_pktmbuf_prefree_seg(txep[0].mbuf);
-	if (likely(m != NULL)) {
-		free[0] = m;
-		nb_free = 1;
-		for (i = 1; i < n; i++) {
-			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
-			if (likely(m != NULL)) {
-				if (likely(m->pool == free[0]->pool)) {
-					free[nb_free++] = m;
-				} else {
-					rte_mempool_put_bulk(free[0]->pool,
-							     (void *)free,
-							     nb_free);
-					free[0] = m;
-					nb_free = 1;
-				}
-			}
-		}
-		rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
-	} else {
-		for (i = 1; i < n; i++) {
-			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
-			if (m != NULL)
-				rte_mempool_put(m->pool, m);
-		}
-	}
-
-	/* buffers were freed, update counters */
-	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
-	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
-	if (txq->tx_next_dd >= txq->nb_tx_desc)
-		txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
-
-	return txq->tx_rs_thresh;
-}
-
-static inline void __attribute__((always_inline))
-tx_backlog_entry(struct i40e_tx_entry *txep,
-		 struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
-{
-	int i;
-
-	for (i = 0; i < (int)nb_pkts; ++i)
-		txep[i].mbuf = tx_pkts[i];
-}
-
 uint16_t
 i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 		   uint16_t nb_pkts)
@@ -709,49 +581,13 @@ i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
 void __attribute__((cold))
 i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)
 {
-	const unsigned mask = rxq->nb_rx_desc - 1;
-	unsigned i;
-
-	if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc)
-		return;
-
-	/* free all mbufs that are valid in the ring */
-	if (rxq->rxrearm_nb == 0) {
-		for (i = 0; i < rxq->nb_rx_desc; i++) {
-			if (rxq->sw_ring[i].mbuf != NULL)
-				rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
-		}
-	} else {
-		for (i = rxq->rx_tail;
-		     i != rxq->rxrearm_start;
-		     i = (i + 1) & mask) {
-			if (rxq->sw_ring[i].mbuf != NULL)
-				rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
-		}
-	}
-
-	rxq->rxrearm_nb = rxq->nb_rx_desc;
-
-	/* set all entries to NULL */
-	memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
+	_i40e_rx_queue_release_mbufs_vec(rxq);
 }
 
 int __attribute__((cold))
 i40e_rxq_vec_setup(struct i40e_rx_queue *rxq)
 {
-	uintptr_t p;
-	struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
-
-	mb_def.nb_segs = 1;
-	mb_def.data_off = RTE_PKTMBUF_HEADROOM;
-	mb_def.port = rxq->port_id;
-	rte_mbuf_refcnt_set(&mb_def, 1);
-
-	/* prevent compiler reordering: rearm_data covers previous fields */
-	rte_compiler_barrier();
-	p = (uintptr_t)&mb_def.rearm_data;
-	rxq->mbuf_initializer = *(uint64_t *)p;
-	return 0;
+	return i40e_rxq_vec_setup_default(rxq);
 }
 
 int __attribute__((cold))
@@ -764,34 +600,10 @@ int __attribute__((cold))
 i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)
 {
 #ifndef RTE_LIBRTE_IEEE1588
-	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
-	struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf;
-
 	/* need SSE4.1 support */
 	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
 		return -1;
-
-#ifndef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
-	/* whithout rx ol_flags, no VP flag report */
-	if (rxmode->hw_vlan_strip != 0 ||
-	    rxmode->hw_vlan_extend != 0 ||
-	    rxmode->hw_ip_checksum != 0)
-		return -1;
 #endif
 
-	/* no fdir support */
-	if (fconf->mode != RTE_FDIR_MODE_NONE)
-		return -1;
-
-	 /* - no csum error report support
-	 * - no header split support
-	 */
-	if (rxmode->header_split == 1)
-		return -1;
-
-	return 0;
-#else
-	RTE_SET_USED(dev);
-	return -1;
-#endif
+	return i40e_rx_vec_dev_conf_condition_check_default(dev);
 }
diff --git a/drivers/net/i40e/i40e_rxtx_vec_common.h b/drivers/net/i40e/i40e_rxtx_vec_common.h
new file mode 100644
index 0000000..6cb5dce
--- /dev/null
+++ b/drivers/net/i40e/i40e_rxtx_vec_common.h
@@ -0,0 +1,251 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _I40E_RXTX_VEC_COMMON_H_
+#define _I40E_RXTX_VEC_COMMON_H_
+#include <stdint.h>
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+
+#include "i40e_ethdev.h"
+#include "i40e_rxtx.h"
+
+static inline uint16_t
+reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
+		   uint16_t nb_bufs, uint8_t *split_flags)
+{
+	struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/
+	struct rte_mbuf *start = rxq->pkt_first_seg;
+	struct rte_mbuf *end =  rxq->pkt_last_seg;
+	unsigned pkt_idx, buf_idx;
+
+	for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
+		if (end != NULL) {
+			/* processing a split packet */
+			end->next = rx_bufs[buf_idx];
+			rx_bufs[buf_idx]->data_len += rxq->crc_len;
+
+			start->nb_segs++;
+			start->pkt_len += rx_bufs[buf_idx]->data_len;
+			end = end->next;
+
+			if (!split_flags[buf_idx]) {
+				/* it's the last packet of the set */
+				start->hash = end->hash;
+				start->ol_flags = end->ol_flags;
+				/* we need to strip crc for the whole packet */
+				start->pkt_len -= rxq->crc_len;
+				if (end->data_len > rxq->crc_len) {
+					end->data_len -= rxq->crc_len;
+				} else {
+					/* free up last mbuf */
+					struct rte_mbuf *secondlast = start;
+
+					while (secondlast->next != end)
+						secondlast = secondlast->next;
+					secondlast->data_len -= (rxq->crc_len -
+							end->data_len);
+					secondlast->next = NULL;
+					rte_pktmbuf_free_seg(end);
+					end = secondlast;
+				}
+				pkts[pkt_idx++] = start;
+				start = end = NULL;
+			}
+		} else {
+			/* not processing a split packet */
+			if (!split_flags[buf_idx]) {
+				/* not a split packet, save and skip */
+				pkts[pkt_idx++] = rx_bufs[buf_idx];
+				continue;
+			}
+			end = start = rx_bufs[buf_idx];
+			rx_bufs[buf_idx]->data_len += rxq->crc_len;
+			rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
+		}
+	}
+
+	/* save the partial packet for next time */
+	rxq->pkt_first_seg = start;
+	rxq->pkt_last_seg = end;
+	memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
+	return pkt_idx;
+}
+
+static inline int __attribute__((always_inline))
+i40e_tx_free_bufs(struct i40e_tx_queue *txq)
+{
+	struct i40e_tx_entry *txep;
+	uint32_t n;
+	uint32_t i;
+	int nb_free = 0;
+	struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
+
+	/* check DD bits on threshold descriptor */
+	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
+			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
+			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
+		return 0;
+
+	n = txq->tx_rs_thresh;
+
+	 /* first buffer to free from S/W ring is at index
+	  * tx_next_dd - (tx_rs_thresh-1)
+	  */
+	txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
+	m = __rte_pktmbuf_prefree_seg(txep[0].mbuf);
+	if (likely(m != NULL)) {
+		free[0] = m;
+		nb_free = 1;
+		for (i = 1; i < n; i++) {
+			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
+			if (likely(m != NULL)) {
+				if (likely(m->pool == free[0]->pool)) {
+					free[nb_free++] = m;
+				} else {
+					rte_mempool_put_bulk(free[0]->pool,
+							     (void *)free,
+							     nb_free);
+					free[0] = m;
+					nb_free = 1;
+				}
+			}
+		}
+		rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
+	} else {
+		for (i = 1; i < n; i++) {
+			m = __rte_pktmbuf_prefree_seg(txep[i].mbuf);
+			if (m != NULL)
+				rte_mempool_put(m->pool, m);
+		}
+	}
+
+	/* buffers were freed, update counters */
+	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
+	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
+	if (txq->tx_next_dd >= txq->nb_tx_desc)
+		txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
+
+	return txq->tx_rs_thresh;
+}
+
+static inline void __attribute__((always_inline))
+tx_backlog_entry(struct i40e_tx_entry *txep,
+		 struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+	int i;
+
+	for (i = 0; i < (int)nb_pkts; ++i)
+		txep[i].mbuf = tx_pkts[i];
+}
+
+static inline void
+_i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)
+{
+	const unsigned mask = rxq->nb_rx_desc - 1;
+	unsigned i;
+
+	if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc)
+		return;
+
+	/* free all mbufs that are valid in the ring */
+	if (rxq->rxrearm_nb == 0) {
+		for (i = 0; i < rxq->nb_rx_desc; i++) {
+			if (rxq->sw_ring[i].mbuf != NULL)
+				rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+		}
+	} else {
+		for (i = rxq->rx_tail;
+		     i != rxq->rxrearm_start;
+		     i = (i + 1) & mask) {
+			if (rxq->sw_ring[i].mbuf != NULL)
+				rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+		}
+	}
+
+	rxq->rxrearm_nb = rxq->nb_rx_desc;
+
+	/* set all entries to NULL */
+	memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
+}
+
+static inline int
+i40e_rxq_vec_setup_default(struct i40e_rx_queue *rxq)
+{
+	uintptr_t p;
+	struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
+
+	mb_def.nb_segs = 1;
+	mb_def.data_off = RTE_PKTMBUF_HEADROOM;
+	mb_def.port = rxq->port_id;
+	rte_mbuf_refcnt_set(&mb_def, 1);
+
+	/* prevent compiler reordering: rearm_data covers previous fields */
+	rte_compiler_barrier();
+	p = (uintptr_t)&mb_def.rearm_data;
+	rxq->mbuf_initializer = *(uint64_t *)p;
+	return 0;
+}
+
+static inline int
+i40e_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev *dev)
+{
+#ifndef RTE_LIBRTE_IEEE1588
+	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
+	struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf;
+
+#ifndef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
+	/* whithout rx ol_flags, no VP flag report */
+	if (rxmode->hw_vlan_strip != 0 ||
+	    rxmode->hw_vlan_extend != 0 ||
+	    rxmode->hw_ip_checksum != 0)
+		return -1;
+#endif
+
+	/* no fdir support */
+	if (fconf->mode != RTE_FDIR_MODE_NONE)
+		return -1;
+
+	 /* - no csum error report support
+	 * - no header split support
+	 */
+	if (rxmode->header_split == 1)
+		return -1;
+
+	return 0;
+#else
+	RTE_SET_USED(dev);
+	return -1;
+#endif
+}
+#endif
-- 
2.4.11

[dpdk-dev] [PATCH v2 2/5] i40e: implement vector PMD for ARM architecture

[Jianbo Liu]

Use ARM NEON intrinsic to implement i40e vPMD

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 drivers/net/i40e/Makefile             |   4 +
 drivers/net/i40e/i40e_rxtx_vec_neon.c | 614 ++++++++++++++++++++++++++++++++++
 2 files changed, 618 insertions(+)
 create mode 100644 drivers/net/i40e/i40e_rxtx_vec_neon.c

diff --git a/drivers/net/i40e/Makefile b/drivers/net/i40e/Makefile
index 53fe145..9e92b38 100644
--- a/drivers/net/i40e/Makefile
+++ b/drivers/net/i40e/Makefile
@@ -97,7 +97,11 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_dcb.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_rxtx.c
+ifeq ($(CONFIG_RTE_ARCH_ARM64),y)
+SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_neon.c
+else
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec.c
+endif
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev_vf.c
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_pf.c
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_fdir.c
diff --git a/drivers/net/i40e/i40e_rxtx_vec_neon.c b/drivers/net/i40e/i40e_rxtx_vec_neon.c
new file mode 100644
index 0000000..011c54e
--- /dev/null
+++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c
@@ -0,0 +1,614 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2016, Linaro Limited
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+
+#include "base/i40e_prototype.h"
+#include "base/i40e_type.h"
+#include "i40e_ethdev.h"
+#include "i40e_rxtx.h"
+#include "i40e_rxtx_vec_common.h"
+
+#include <arm_neon.h>
+
+#pragma GCC diagnostic ignored "-Wcast-qual"
+
+static inline void
+i40e_rxq_rearm(struct i40e_rx_queue *rxq)
+{
+	int i;
+	uint16_t rx_id;
+	volatile union i40e_rx_desc *rxdp;
+	struct i40e_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
+	struct rte_mbuf *mb0, *mb1;
+	uint64x2_t dma_addr0, dma_addr1;
+	uint64x2_t zero = vdupq_n_u64(0);
+	uint64_t paddr;
+	uint8x8_t p;
+
+	rxdp = rxq->rx_ring + rxq->rxrearm_start;
+
+	/* Pull 'n' more MBUFs into the software ring */
+	if (unlikely(rte_mempool_get_bulk(rxq->mp,
+					  (void *)rxep,
+					  RTE_I40E_RXQ_REARM_THRESH) < 0)) {
+		if (rxq->rxrearm_nb + RTE_I40E_RXQ_REARM_THRESH >=
+		    rxq->nb_rx_desc) {
+			for (i = 0; i < RTE_I40E_DESCS_PER_LOOP; i++) {
+				rxep[i].mbuf = &rxq->fake_mbuf;
+				vst1q_u64((uint64_t *)&rxdp[i].read, zero);
+			}
+		}
+		rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
+			RTE_I40E_RXQ_REARM_THRESH;
+		return;
+	}
+
+	p = vld1_u8((uint8_t *)&rxq->mbuf_initializer);
+
+	/* Initialize the mbufs in vector, process 2 mbufs in one loop */
+	for (i = 0; i < RTE_I40E_RXQ_REARM_THRESH; i += 2, rxep += 2) {
+		mb0 = rxep[0].mbuf;
+		mb1 = rxep[1].mbuf;
+
+		 /* Flush mbuf with pkt template.
+		 * Data to be rearmed is 6 bytes long.
+		 * Though, RX will overwrite ol_flags that are coming next
+		 * anyway. So overwrite whole 8 bytes with one load:
+		 * 6 bytes of rearm_data plus first 2 bytes of ol_flags.
+		 */
+		vst1_u8((uint8_t *)&mb0->rearm_data, p);
+		paddr = mb0->buf_physaddr + RTE_PKTMBUF_HEADROOM;
+		dma_addr0 = vdupq_n_u64(paddr);
+
+		/* flush desc with pa dma_addr */
+		vst1q_u64((uint64_t *)&rxdp++->read, dma_addr0);
+
+		vst1_u8((uint8_t *)&mb1->rearm_data, p);
+		paddr = mb1->buf_physaddr + RTE_PKTMBUF_HEADROOM;
+		dma_addr1 = vdupq_n_u64(paddr);
+		vst1q_u64((uint64_t *)&rxdp++->read, dma_addr1);
+	}
+
+	rxq->rxrearm_start += RTE_I40E_RXQ_REARM_THRESH;
+	if (rxq->rxrearm_start >= rxq->nb_rx_desc)
+		rxq->rxrearm_start = 0;
+
+	rxq->rxrearm_nb -= RTE_I40E_RXQ_REARM_THRESH;
+
+	rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
+			     (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
+
+	/* Update the tail pointer on the NIC */
+	I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
+}
+
+/* Handling the offload flags (olflags) field takes computation
+ * time when receiving packets. Therefore we provide a flag to disable
+ * the processing of the olflags field when they are not needed. This
+ * gives improved performance, at the cost of losing the offload info
+ * in the received packet
+ */
+#ifdef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
+
+static inline void
+desc_to_olflags_v(uint64x2_t descs[4], struct rte_mbuf **rx_pkts)
+{
+	uint32x4_t vlan0, vlan1, rss, l3_l4e;
+
+	/* mask everything except RSS, flow director and VLAN flags
+	 * bit2 is for VLAN tag, bit11 for flow director indication
+	 * bit13:12 for RSS indication.
+	 */
+	const uint32x4_t rss_vlan_msk = {
+			0x1c03804, 0x1c03804, 0x1c03804, 0x1c03804};
+
+	/* map rss and vlan type to rss hash and vlan flag */
+	const uint8x16_t vlan_flags = {
+			0, 0, 0, 0,
+			PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED, 0, 0, 0,
+			0, 0, 0, 0,
+			0, 0, 0, 0};
+
+	const uint8x16_t rss_flags = {
+			0, PKT_RX_FDIR, 0, 0,
+			0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR,
+			0, 0, 0, 0,
+			0, 0, 0, 0};
+
+	const uint8x16_t l3_l4e_flags = {
+			0,
+			PKT_RX_IP_CKSUM_BAD,
+			PKT_RX_L4_CKSUM_BAD,
+			PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD,
+			PKT_RX_EIP_CKSUM_BAD,
+			PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD,
+			PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD,
+			PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD,
+			0, 0, 0, 0, 0, 0, 0, 0};
+
+	vlan0 = vzipq_u32(vreinterpretq_u32_u64(descs[0]),
+			  vreinterpretq_u32_u64(descs[2])).val[1];
+	vlan1 = vzipq_u32(vreinterpretq_u32_u64(descs[1]),
+			  vreinterpretq_u32_u64(descs[3])).val[1];
+	vlan0 = vzipq_u32(vlan0, vlan1).val[0];
+
+	vlan1 = vandq_u32(vlan0, rss_vlan_msk);
+	vlan0 = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags,
+						vreinterpretq_u8_u32(vlan1)));
+
+	rss = vshrq_n_u32(vlan1, 11);
+	rss = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags,
+					      vreinterpretq_u8_u32(rss)));
+
+	l3_l4e = vshrq_n_u32(vlan1, 22);
+	l3_l4e = vreinterpretq_u32_u8(vqtbl1q_u8(l3_l4e_flags,
+					      vreinterpretq_u8_u32(l3_l4e)));
+
+
+	vlan0 = vorrq_u32(vlan0, rss);
+	vlan0 = vorrq_u32(vlan0, l3_l4e);
+
+	rx_pkts[0]->ol_flags = vgetq_lane_u32(vlan0, 0);
+	rx_pkts[1]->ol_flags = vgetq_lane_u32(vlan0, 1);
+	rx_pkts[2]->ol_flags = vgetq_lane_u32(vlan0, 2);
+	rx_pkts[3]->ol_flags = vgetq_lane_u32(vlan0, 3);
+}
+#else
+#define desc_to_olflags_v(descs, rx_pkts) do {} while (0)
+#endif
+
+#define PKTLEN_SHIFT     10
+
+#define I40E_VPMD_DESC_DD_MASK	0x0001000100010001ULL
+
+static inline void
+desc_to_ptype_v(uint64x2_t descs[4], struct rte_mbuf **rx_pkts)
+{
+	int i;
+	uint8_t ptype;
+	uint8x16_t tmp;
+
+	for (i = 0; i < 4; i++) {
+		tmp = vreinterpretq_u8_u64(vshrq_n_u64(descs[i], 30));
+		ptype = vgetq_lane_u8(tmp, 8);
+		rx_pkts[0]->packet_type = i40e_rxd_pkt_type_mapping(ptype);
+	}
+
+}
+
+ /*
+ * Notice:
+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
+ *   numbers of DD bits
+ */
+static inline uint16_t
+_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+		   uint16_t nb_pkts, uint8_t *split_packet)
+{
+	volatile union i40e_rx_desc *rxdp;
+	struct i40e_rx_entry *sw_ring;
+	uint16_t nb_pkts_recd;
+	int pos;
+	uint64_t var;
+
+	/* mask to shuffle from desc. to mbuf */
+	uint8x16_t shuf_msk = {
+		0xFF, 0xFF,   /* pkt_type set as unknown */
+		0xFF, 0xFF,   /* pkt_type set as unknown */
+		14, 15,       /* octet 15~14, low 16 bits pkt_len */
+		0xFF, 0xFF,   /* skip high 16 bits pkt_len, zero out */
+		14, 15,       /* octet 15~14, 16 bits data_len */
+		2, 3,         /* octet 2~3, low 16 bits vlan_macip */
+		4, 5, 6, 7    /* octet 4~7, 32bits rss */
+		};
+
+	uint8x16_t eop_check = {
+		0x02, 0x00, 0x02, 0x00,
+		0x02, 0x00, 0x02, 0x00,
+		0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00
+		};
+
+	uint16x8_t crc_adjust = {
+		0, 0,         /* ignore pkt_type field */
+		rxq->crc_len, /* sub crc on pkt_len */
+		0,            /* ignore high-16bits of pkt_len */
+		rxq->crc_len, /* sub crc on data_len */
+		0, 0, 0       /* ignore non-length fields */
+		};
+
+	/* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */
+	nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST);
+
+	/* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */
+	nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP);
+
+	/* Just the act of getting into the function from the application is
+	 * going to cost about 7 cycles
+	 */
+	rxdp = rxq->rx_ring + rxq->rx_tail;
+
+	rte_prefetch_non_temporal(rxdp);
+
+	/* See if we need to rearm the RX queue - gives the prefetch a bit
+	 * of time to act
+	 */
+	if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH)
+		i40e_rxq_rearm(rxq);
+
+	/* Before we start moving massive data around, check to see if
+	 * there is actually a packet available
+	 */
+	if (!(rxdp->wb.qword1.status_error_len &
+			rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
+		return 0;
+
+	/* Cache is empty -> need to scan the buffer rings, but first move
+	 * the next 'n' mbufs into the cache
+	 */
+	sw_ring = &rxq->sw_ring[rxq->rx_tail];
+
+	/* A. load 4 packet in one loop
+	 * [A*. mask out 4 unused dirty field in desc]
+	 * B. copy 4 mbuf point from swring to rx_pkts
+	 * C. calc the number of DD bits among the 4 packets
+	 * [C*. extract the end-of-packet bit, if requested]
+	 * D. fill info. from desc to mbuf
+	 */
+
+	for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
+			pos += RTE_I40E_DESCS_PER_LOOP,
+			rxdp += RTE_I40E_DESCS_PER_LOOP) {
+		uint64x2_t descs[RTE_I40E_DESCS_PER_LOOP];
+		uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+		uint16x8x2_t sterr_tmp1, sterr_tmp2;
+		uint64x2_t mbp1, mbp2;
+		uint16x8_t staterr;
+		uint16x8_t tmp;
+		uint64_t stat;
+
+		int32x4_t len_shl = {0, 0, 0, PKTLEN_SHIFT};
+
+		/* B.1 load 1 mbuf point */
+		mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
+		/* Read desc statuses backwards to avoid race condition */
+		/* A.1 load 4 pkts desc */
+		descs[3] =  vld1q_u64((uint64_t *)(rxdp + 3));
+		rte_rmb();
+
+		/* B.2 copy 2 mbuf point into rx_pkts  */
+		vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1);
+
+		/* B.1 load 1 mbuf point */
+		mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
+
+		descs[2] =  vld1q_u64((uint64_t *)(rxdp + 2));
+		/* B.1 load 2 mbuf point */
+		descs[1] =  vld1q_u64((uint64_t *)(rxdp + 1));
+		descs[0] =  vld1q_u64((uint64_t *)(rxdp));
+
+		/* B.2 copy 2 mbuf point into rx_pkts  */
+		vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2);
+
+		if (split_packet) {
+			rte_mbuf_prefetch_part2(rx_pkts[pos]);
+			rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
+			rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
+			rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
+		}
+
+		/* avoid compiler reorder optimization */
+		rte_compiler_barrier();
+
+		/* pkt 3,4 shift the pktlen field to be 16-bit aligned*/
+		uint32x4_t len3 = vshlq_u32(vreinterpretq_u32_u64(descs[3]),
+					    len_shl);
+		descs[3] = vreinterpretq_u64_u32(len3);
+		uint32x4_t len2 = vshlq_u32(vreinterpretq_u32_u64(descs[2]),
+					    len_shl);
+		descs[2] = vreinterpretq_u64_u32(len2);
+
+		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
+		pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk);
+		pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk);
+
+		/* C.1 4=>2 filter staterr info only */
+		sterr_tmp2 = vzipq_u16(vreinterpretq_u16_u64(descs[1]),
+				       vreinterpretq_u16_u64(descs[3]));
+		/* C.1 4=>2 filter staterr info only */
+		sterr_tmp1 = vzipq_u16(vreinterpretq_u16_u64(descs[0]),
+				       vreinterpretq_u16_u64(descs[2]));
+
+		/* C.2 get 4 pkts staterr value  */
+		staterr = vzipq_u16(sterr_tmp1.val[1],
+				    sterr_tmp2.val[1]).val[0];
+		stat = vgetq_lane_u64(vreinterpretq_u64_u16(staterr), 0);
+
+		desc_to_olflags_v(descs, &rx_pkts[pos]);
+
+		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
+		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
+		pkt_mb4 = vreinterpretq_u8_u16(tmp);
+		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
+		pkt_mb3 = vreinterpretq_u8_u16(tmp);
+
+		/* pkt 1,2 shift the pktlen field to be 16-bit aligned*/
+		uint32x4_t len1 = vshlq_u32(vreinterpretq_u32_u64(descs[1]),
+					    len_shl);
+		descs[1] = vreinterpretq_u64_u32(len1);
+		uint32x4_t len0 = vshlq_u32(vreinterpretq_u32_u64(descs[0]),
+					    len_shl);
+		descs[0] = vreinterpretq_u64_u32(len0);
+
+		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
+		pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), shuf_msk);
+		pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), shuf_msk);
+
+		/* D.3 copy final 3,4 data to rx_pkts */
+		vst1q_u8((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
+				 pkt_mb4);
+		vst1q_u8((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
+				 pkt_mb3);
+
+		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
+		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
+		pkt_mb2 = vreinterpretq_u8_u16(tmp);
+		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
+		pkt_mb1 = vreinterpretq_u8_u16(tmp);
+
+		/* C* extract and record EOP bit */
+		if (split_packet) {
+			uint8x16_t eop_shuf_mask = {
+					0x00, 0x02, 0x04, 0x06,
+					0xFF, 0xFF, 0xFF, 0xFF,
+					0xFF, 0xFF, 0xFF, 0xFF,
+					0xFF, 0xFF, 0xFF, 0xFF};
+			uint8x16_t eop_bits;
+
+			/* and with mask to extract bits, flipping 1-0 */
+			eop_bits = vmvnq_u8(vreinterpretq_u8_u16(staterr));
+			eop_bits = vandq_u8(eop_bits, eop_check);
+			/* the staterr values are not in order, as the count
+			 * count of dd bits doesn't care. However, for end of
+			 * packet tracking, we do care, so shuffle. This also
+			 * compresses the 32-bit values to 8-bit
+			 */
+			eop_bits = vqtbl1q_u8(eop_bits, eop_shuf_mask);
+
+			/* store the resulting 32-bit value */
+			vst1q_lane_u32((uint32_t *)split_packet,
+				       vreinterpretq_u32_u8(eop_bits), 0);
+			split_packet += RTE_I40E_DESCS_PER_LOOP;
+
+			/* zero-out next pointers */
+			rx_pkts[pos]->next = NULL;
+			rx_pkts[pos + 1]->next = NULL;
+			rx_pkts[pos + 2]->next = NULL;
+			rx_pkts[pos + 3]->next = NULL;
+		}
+
+		rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP);
+
+		/* D.3 copy final 1,2 data to rx_pkts */
+		vst1q_u8((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
+			 pkt_mb2);
+		vst1q_u8((void *)&rx_pkts[pos]->rx_descriptor_fields1,
+			 pkt_mb1);
+		desc_to_ptype_v(descs, &rx_pkts[pos]);
+		/* C.4 calc avaialbe number of desc */
+		var = __builtin_popcountll(stat & I40E_VPMD_DESC_DD_MASK);
+		nb_pkts_recd += var;
+		if (likely(var != RTE_I40E_DESCS_PER_LOOP))
+			break;
+	}
+
+	/* Update our internal tail pointer */
+	rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);
+	rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));
+	rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
+
+	return nb_pkts_recd;
+}
+
+ /*
+ * Notice:
+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
+ *   numbers of DD bits
+ */
+uint16_t
+i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+		   uint16_t nb_pkts)
+{
+	return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
+}
+
+ /* vPMD receive routine that reassembles scattered packets
+ * Notice:
+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
+ *   numbers of DD bits
+ */
+uint16_t
+i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+			     uint16_t nb_pkts)
+{
+
+	struct i40e_rx_queue *rxq = rx_queue;
+	uint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0};
+
+	/* get some new buffers */
+	uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts,
+			split_flags);
+	if (nb_bufs == 0)
+		return 0;
+
+	/* happy day case, full burst + no packets to be joined */
+	const uint64_t *split_fl64 = (uint64_t *)split_flags;
+
+	if (rxq->pkt_first_seg == NULL &&
+			split_fl64[0] == 0 && split_fl64[1] == 0 &&
+			split_fl64[2] == 0 && split_fl64[3] == 0)
+		return nb_bufs;
+
+	/* reassemble any packets that need reassembly*/
+	unsigned i = 0;
+
+	if (rxq->pkt_first_seg == NULL) {
+		/* find the first split flag, and only reassemble then*/
+		while (i < nb_bufs && !split_flags[i])
+			i++;
+		if (i == nb_bufs)
+			return nb_bufs;
+	}
+	return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,
+		&split_flags[i]);
+}
+
+static inline void
+vtx1(volatile struct i40e_tx_desc *txdp,
+		struct rte_mbuf *pkt, uint64_t flags)
+{
+	uint64_t high_qw = (I40E_TX_DESC_DTYPE_DATA |
+			((uint64_t)flags  << I40E_TXD_QW1_CMD_SHIFT) |
+			((uint64_t)pkt->data_len << I40E_TXD_QW1_TX_BUF_SZ_SHIFT));
+
+	uint64x2_t descriptor = {pkt->buf_physaddr + pkt->data_off, high_qw};
+	vst1q_u64((uint64_t *)txdp, descriptor);
+}
+
+static inline void
+vtx(volatile struct i40e_tx_desc *txdp,
+		struct rte_mbuf **pkt, uint16_t nb_pkts,  uint64_t flags)
+{
+	int i;
+
+	for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt)
+		vtx1(txdp, *pkt, flags);
+}
+
+uint16_t
+i40e_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+		   uint16_t nb_pkts)
+{
+	struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;
+	volatile struct i40e_tx_desc *txdp;
+	struct i40e_tx_entry *txep;
+	uint16_t n, nb_commit, tx_id;
+	uint64_t flags = I40E_TD_CMD;
+	uint64_t rs = I40E_TX_DESC_CMD_RS | I40E_TD_CMD;
+	int i;
+
+	/* cross rx_thresh boundary is not allowed */
+	nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh);
+
+	if (txq->nb_tx_free < txq->tx_free_thresh)
+		i40e_tx_free_bufs(txq);
+
+	nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
+	if (unlikely(nb_pkts == 0))
+		return 0;
+
+	tx_id = txq->tx_tail;
+	txdp = &txq->tx_ring[tx_id];
+	txep = &txq->sw_ring[tx_id];
+
+	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
+
+	n = (uint16_t)(txq->nb_tx_desc - tx_id);
+	if (nb_commit >= n) {
+		tx_backlog_entry(txep, tx_pkts, n);
+
+		for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp)
+			vtx1(txdp, *tx_pkts, flags);
+
+		vtx1(txdp, *tx_pkts++, rs);
+
+		nb_commit = (uint16_t)(nb_commit - n);
+
+		tx_id = 0;
+		txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
+
+		/* avoid reach the end of ring */
+		txdp = &txq->tx_ring[tx_id];
+		txep = &txq->sw_ring[tx_id];
+	}
+
+	tx_backlog_entry(txep, tx_pkts, nb_commit);
+
+	vtx(txdp, tx_pkts, nb_commit, flags);
+
+	tx_id = (uint16_t)(tx_id + nb_commit);
+	if (tx_id > txq->tx_next_rs) {
+		txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |=
+			rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
+						I40E_TXD_QW1_CMD_SHIFT);
+		txq->tx_next_rs =
+			(uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
+	}
+
+	txq->tx_tail = tx_id;
+
+	I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
+
+	return nb_pkts;
+}
+
+void __attribute__((cold))
+i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)
+{
+	_i40e_rx_queue_release_mbufs_vec(rxq);
+}
+
+int __attribute__((cold))
+i40e_rxq_vec_setup(struct i40e_rx_queue *rxq)
+{
+	return i40e_rxq_vec_setup_default(rxq);
+}
+
+int __attribute__((cold))
+i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq)
+{
+	return 0;
+}
+
+int __attribute__((cold))
+i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)
+{
+	return i40e_rx_vec_dev_conf_condition_check_default(dev);
+}
-- 
2.4.11

Re: [dpdk-dev] [PATCH v2 2/5] i40e: implement vector PMD for ARM architecture

[Jerin Jacob]

On Fri, Oct 14, 2016 at 09:30:01AM +0530, Jianbo Liu wrote:
> Use ARM NEON intrinsic to implement i40e vPMD
> 
> Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>

I'm not entirely familiar with i40e internals.The patch looks OK interms
of using NEON instructions.

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

> ---
>  drivers/net/i40e/Makefile             |   4 +
>  drivers/net/i40e/i40e_rxtx_vec_neon.c | 614 ++++++++++++++++++++++++++++++++++
>  2 files changed, 618 insertions(+)
>  create mode 100644 drivers/net/i40e/i40e_rxtx_vec_neon.c
> 

[dpdk-dev] [PATCH v2 3/5] i40e: enable i40e vector PMD on ARMv8a platform

[Jianbo Liu]

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 config/defconfig_arm64-armv8a-linuxapp-gcc | 1 -
 doc/guides/nics/features/i40e_vec.ini      | 1 +
 doc/guides/nics/features/i40e_vf_vec.ini   | 1 +
 3 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/defconfig_arm64-armv8a-linuxapp-gcc b/config/defconfig_arm64-armv8a-linuxapp-gcc
index a0f4473..6321884 100644
--- a/config/defconfig_arm64-armv8a-linuxapp-gcc
+++ b/config/defconfig_arm64-armv8a-linuxapp-gcc
@@ -45,6 +45,5 @@ CONFIG_RTE_TOOLCHAIN_GCC=y
 CONFIG_RTE_EAL_IGB_UIO=n
 
 CONFIG_RTE_LIBRTE_FM10K_PMD=n
-CONFIG_RTE_LIBRTE_I40E_INC_VECTOR=n
 
 CONFIG_RTE_SCHED_VECTOR=n
diff --git a/doc/guides/nics/features/i40e_vec.ini b/doc/guides/nics/features/i40e_vec.ini
index 0953d84..edd6b71 100644
--- a/doc/guides/nics/features/i40e_vec.ini
+++ b/doc/guides/nics/features/i40e_vec.ini
@@ -37,3 +37,4 @@ Linux UIO            = Y
 Linux VFIO           = Y
 x86-32               = Y
 x86-64               = Y
+ARMv8                = Y
diff --git a/doc/guides/nics/features/i40e_vf_vec.ini b/doc/guides/nics/features/i40e_vf_vec.ini
index 2a44bf6..d6674f7 100644
--- a/doc/guides/nics/features/i40e_vf_vec.ini
+++ b/doc/guides/nics/features/i40e_vf_vec.ini
@@ -26,3 +26,4 @@ Linux UIO            = Y
 Linux VFIO           = Y
 x86-32               = Y
 x86-64               = Y
+ARMv8                = Y
-- 
2.4.11

Re: [dpdk-dev] [PATCH v2 3/5] i40e: enable i40e vector PMD on ARMv8a platform

[Jerin Jacob]

On Fri, Oct 14, 2016 at 09:30:02AM +0530, Jianbo Liu wrote:
> Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>

Reviewed-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

> ---
>  config/defconfig_arm64-armv8a-linuxapp-gcc | 1 -
>  doc/guides/nics/features/i40e_vec.ini      | 1 +
>  doc/guides/nics/features/i40e_vf_vec.ini   | 1 +
>  3 files changed, 2 insertions(+), 1 deletion(-)
> 
> diff --git a/config/defconfig_arm64-armv8a-linuxapp-gcc b/config/defconfig_arm64-armv8a-linuxapp-gcc
> index a0f4473..6321884 100644
> --- a/config/defconfig_arm64-armv8a-linuxapp-gcc
> +++ b/config/defconfig_arm64-armv8a-linuxapp-gcc
> @@ -45,6 +45,5 @@ CONFIG_RTE_TOOLCHAIN_GCC=y
>  CONFIG_RTE_EAL_IGB_UIO=n
>  
>  CONFIG_RTE_LIBRTE_FM10K_PMD=n
> -CONFIG_RTE_LIBRTE_I40E_INC_VECTOR=n
>  
>  CONFIG_RTE_SCHED_VECTOR=n
> diff --git a/doc/guides/nics/features/i40e_vec.ini b/doc/guides/nics/features/i40e_vec.ini
> index 0953d84..edd6b71 100644
> --- a/doc/guides/nics/features/i40e_vec.ini
> +++ b/doc/guides/nics/features/i40e_vec.ini
> @@ -37,3 +37,4 @@ Linux UIO            = Y
>  Linux VFIO           = Y
>  x86-32               = Y
>  x86-64               = Y
> +ARMv8                = Y
> diff --git a/doc/guides/nics/features/i40e_vf_vec.ini b/doc/guides/nics/features/i40e_vf_vec.ini
> index 2a44bf6..d6674f7 100644
> --- a/doc/guides/nics/features/i40e_vf_vec.ini
> +++ b/doc/guides/nics/features/i40e_vf_vec.ini
> @@ -26,3 +26,4 @@ Linux UIO            = Y
>  Linux VFIO           = Y
>  x86-32               = Y
>  x86-64               = Y
> +ARMv8                = Y
> -- 
> 2.4.11
> 

[dpdk-dev] [PATCH v2 4/5] i40e: make vector driver filenames consistent

[Jianbo Liu]

To be consistent with the naming for ARM NEON implementation,
i40e_rxtx_vec.c is renamed to i40e_rxtx_vec_sse.c.

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 drivers/net/i40e/Makefile                                 | 4 ++--
 drivers/net/i40e/{i40e_rxtx_vec.c => i40e_rxtx_vec_sse.c} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename drivers/net/i40e/{i40e_rxtx_vec.c => i40e_rxtx_vec_sse.c} (100%)

diff --git a/drivers/net/i40e/Makefile b/drivers/net/i40e/Makefile
index 9e92b38..13085fb 100644
--- a/drivers/net/i40e/Makefile
+++ b/drivers/net/i40e/Makefile
@@ -100,7 +100,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_rxtx.c
 ifeq ($(CONFIG_RTE_ARCH_ARM64),y)
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_neon.c
 else
-SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec.c
+SRCS-$(CONFIG_RTE_LIBRTE_I40E_INC_VECTOR) += i40e_rxtx_vec_sse.c
 endif
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_ethdev_vf.c
 SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_pf.c
@@ -108,7 +108,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_I40E_PMD) += i40e_fdir.c
 
 # vector PMD driver needs SSE4.1 support
 ifeq ($(findstring RTE_MACHINE_CPUFLAG_SSE4_1,$(CFLAGS)),)
-CFLAGS_i40e_rxtx_vec.o += -msse4.1
+CFLAGS_i40e_rxtx_vec_sse.o += -msse4.1
 endif
 
 
diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec_sse.c
similarity index 100%
rename from drivers/net/i40e/i40e_rxtx_vec.c
rename to drivers/net/i40e/i40e_rxtx_vec_sse.c
-- 
2.4.11

[dpdk-dev] [PATCH v2 5/5] maintainers: claim i40e vector PMD on ARM

[Jianbo Liu]

Signed-off-by: Jianbo Liu <jianbo.liu@linaro.org>
---
 MAINTAINERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 8f5fa82..621bda6 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -151,6 +151,7 @@ F: lib/librte_acl/acl_run_neon.*
 F: lib/librte_lpm/rte_lpm_neon.h
 F: lib/librte_hash/rte*_arm64.h
 F: drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c
+F: drivers/net/i40e/i40e_rxtx_vec_neon.c
 F: drivers/net/virtio/virtio_rxtx_simple_neon.c
 
 EZchip TILE-Gx
-- 
2.4.11

Re: [dpdk-dev] [PATCH v2 0/5] i40e: vector poll-mode driver on ARM64

[Bruce Richardson]

On Fri, Oct 14, 2016 at 09:29:59AM +0530, Jianbo Liu wrote:
> This patch set is to implement i40e vector PMD on ARM64.
> For x86, vPMD is only reorganized, there should be no performance loss.
> 
> v1 -> v2
> - rebase to dpdk-next-net/rel_16_11
> 
> Jianbo Liu (5):
>   i40e: extract non-x86 specific code from vector driver
>   i40e: implement vector PMD for ARM architecture
>   i40e: enable i40e vector PMD on ARMv8a platform
>   i40e: make vector driver filenames consistent
>   maintainers: claim i40e vector PMD on ARM
> 
While I haven't reviewed the ARM code, I have run a very quick sanity
check on an IA platform and saw no performance regression there.

Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Re: [dpdk-dev] [PATCH v2 0/5] i40e: vector poll-mode driver on ARM64

[Bruce Richardson]

On Mon, Oct 17, 2016 at 03:44:34PM +0100, Bruce Richardson wrote:
> On Fri, Oct 14, 2016 at 09:29:59AM +0530, Jianbo Liu wrote:
> > This patch set is to implement i40e vector PMD on ARM64.
> > For x86, vPMD is only reorganized, there should be no performance loss.
> > 
> > v1 -> v2
> > - rebase to dpdk-next-net/rel_16_11
> > 
> > Jianbo Liu (5):
> >   i40e: extract non-x86 specific code from vector driver
> >   i40e: implement vector PMD for ARM architecture
> >   i40e: enable i40e vector PMD on ARMv8a platform
> >   i40e: make vector driver filenames consistent
> >   maintainers: claim i40e vector PMD on ARM
> > 
> While I haven't reviewed the ARM code, I have run a very quick sanity
> check on an IA platform and saw no performance regression there.
> 
> Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Applied to dpdk-next-net/rel_16_11

/Bruce