[PATCH v1] net/i40e: add flow mark capability to NEON vector routine

[PATCH v1] net/i40e: add flow mark capability to NEON vector routine

[Joyce Kong]

This commit adds a flow director support to i40e NEON
vector RX path.

I40e can have 16 and 32 byte descriptors, and the Flow
Director ID data and indication-bit are in different
locations for each size descriptor. The support is
implemented in two separate functions as they require
vastly different operations.

The 16B descriptor re-purposes the "filter-status" u32
field to indicate FDIR ID when the FLM bit is set. No
extra loads are required, however we do have to store
to mbuf->fdir.hi, which is not stored to in the RX path
before this patch.

The 32B descriptor requires loading the 2nd 16 bytes of
each descriptor, to get the FLEXBH_STAT and FD Filter ID
from qword3. The resulting data must also be stored to
mbuf->fdir.hi, same as the 16B code path.

Signed-off-by: Joyce Kong <joyce.kong@arm.com>
Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
---
 drivers/net/i40e/i40e_rxtx_vec_neon.c | 145 ++++++++++++++++++++++++--
 1 file changed, 139 insertions(+), 6 deletions(-)

diff --git a/drivers/net/i40e/i40e_rxtx_vec_neon.c b/drivers/net/i40e/i40e_rxtx_vec_neon.c
index b951ea2dc3..a7ca2c18e5 100644
--- a/drivers/net/i40e/i40e_rxtx_vec_neon.c
+++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c
@@ -77,9 +77,130 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
 	I40E_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rx_id);
 }
 
+#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
+/* NEON version of FDIR mark extraction for 4 32B descriptors at a time */
+static inline uint32x4_t
+descs_to_fdir_32b(volatile union i40e_rx_desc *rxdp, struct rte_mbuf **rx_pkt)
+{
+	/* 32B descriptors: Load 2nd half of descriptors for FDIR ID data */
+	uint64x2_t desc0_qw23, desc1_qw23, desc2_qw23, desc3_qw23;
+	desc0_qw23 = vld1q_u64((uint64_t *)&(rxdp + 0)->wb.qword2);
+	desc1_qw23 = vld1q_u64((uint64_t *)&(rxdp + 1)->wb.qword2);
+	desc2_qw23 = vld1q_u64((uint64_t *)&(rxdp + 2)->wb.qword2);
+	desc3_qw23 = vld1q_u64((uint64_t *)&(rxdp + 3)->wb.qword2);
+
+	/* FDIR ID data: move last u32 of each desc to 4 u32 lanes */
+	uint32x4_t v_unpack_02, v_unpack_13;
+	v_unpack_02 = vzipq_u32(vreinterpretq_u32_u64(desc0_qw23),
+				vreinterpretq_u32_u64(desc2_qw23)).val[1];
+	v_unpack_13 = vzipq_u32(vreinterpretq_u32_u64(desc1_qw23),
+				vreinterpretq_u32_u64(desc3_qw23)).val[1];
+	uint32x4_t v_fdir_ids = vzipq_u32(v_unpack_02, v_unpack_13).val[1];
+
+	/* Extended Status: extract from each lower 32 bits, to u32 lanes */
+	v_unpack_02 = vzipq_u32(vreinterpretq_u32_u64(desc0_qw23),
+				vreinterpretq_u32_u64(desc2_qw23)).val[0];
+	v_unpack_13 = vzipq_u32(vreinterpretq_u32_u64(desc1_qw23),
+				vreinterpretq_u32_u64(desc3_qw23)).val[0];
+	uint32x4_t v_flt_status = vzipq_u32(v_unpack_02, v_unpack_13).val[0];
+
+	/* Shift u32 left and right to "mask away" bits not required.
+	 * Data required is 4:5 (zero based), so left shift by 26 (32-6)
+	 * and then right shift by 30 (32 - 2 bits required).
+	 */
+	v_flt_status = vshlq_n_u32(v_flt_status, 26);
+	v_flt_status = vshrq_n_u32(v_flt_status, 30);
+
+	/* Generate constant 1 in all u32 lanes */
+	RTE_BUILD_BUG_ON(I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID != 1);
+	uint32x4_t v_u32_one = vdupq_n_u32(1);
+
+	/* Per desc mask, bits set if FDIR ID is valid */
+	uint32x4_t v_fd_id_mask = vceqq_u32(v_flt_status, v_u32_one);
+
+	/* Mask ID data to zero if the FD_ID bit not set in desc */
+	v_fdir_ids = vandq_u32(v_fdir_ids, v_fd_id_mask);
+
+	/* Store data to fdir.hi in mbuf */
+	rx_pkt[0]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 0);
+	rx_pkt[1]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 1);
+	rx_pkt[2]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 2);
+	rx_pkt[3]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 3);
+
+	/* Convert fdir_id_mask into a single bit, then shift as required for
+	 * correct location in the mbuf->olflags
+	 */
+	RTE_BUILD_BUG_ON(RTE_MBUF_F_RX_FDIR_ID != (1 << 13));
+	v_fd_id_mask = vshrq_n_u32(v_fd_id_mask, 31);
+	v_fd_id_mask = vshlq_n_u32(v_fd_id_mask, 13);
+
+	/* The returned value must be combined into each mbuf. This is already
+	 * being done for RSS and VLAN mbuf olflags, so return bits to OR in.
+	 */
+	return v_fd_id_mask;
+}
+
+#else /* 32 or 16B FDIR ID handling */
+
+/* Handle 16B descriptor FDIR ID flag setting based on FLM(bit11). See scalar driver
+ * for scalar implementation of the same functionality.
+ */
+static inline uint32x4_t
+descs_to_fdir_16b(uint32x4_t fltstat, uint64x2_t descs[4], struct rte_mbuf **rx_pkt)
+{
+	/* Unpack filter-status data from descriptors */
+	uint32x4_t v_tmp_02 = vzipq_u32(vreinterpretq_u32_u64(descs[0]),
+					vreinterpretq_u32_u64(descs[2])).val[0];
+	uint32x4_t v_tmp_13 = vzipq_u32(vreinterpretq_u32_u64(descs[1]),
+					vreinterpretq_u32_u64(descs[3])).val[0];
+	uint32x4_t v_fdir_ids = vzipq_u32(v_tmp_02, v_tmp_13).val[1];
+
+	/* Generate 111 and 11 in each u32 lane */
+	uint32x4_t v_111_mask = vdupq_n_u32(7);
+	uint32x4_t v_11_mask = vdupq_n_u32(3);
+
+	/* Compare and mask away FDIR ID data if bit not set */
+	uint32x4_t v_u32_bits = vandq_u32(v_111_mask, fltstat);
+	uint32x4_t v_fdir_id_mask = vceqq_u32(v_u32_bits, v_11_mask);
+	v_fdir_ids = vandq_u32(v_fdir_id_mask, v_fdir_ids);
+
+	/* Store data to fdir.hi in mbuf */
+	rx_pkt[0]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 0);
+	rx_pkt[1]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 1);
+	rx_pkt[2]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 2);
+	rx_pkt[3]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 3);
+
+	/* Top lane ones mask for FDIR isolation */
+	uint32x4_t v_desc_fdir_mask = {0, UINT32_MAX, 0, 0};
+
+	/* Move fdir_id_mask to correct lane, zero RSS in mbuf if fdir hits */
+	uint32x4_t v_zeros = {0, 0, 0, 0};
+	uint32x4_t v_desc3_shift = vextq_u32(v_fdir_id_mask, v_zeros, 2);
+	uint32x4_t v_desc3_mask = vandq_u32(v_desc_fdir_mask, v_desc3_shift);
+	descs[3] = vbslq_u32(v_desc3_mask, v_zeros, vreinterpretq_u32_u64(descs[3]));
+
+	uint32x4_t v_desc2_shift = vextq_u32(v_fdir_id_mask, v_zeros, 1);
+	uint32x4_t v_desc2_mask = vandq_u32(v_desc_fdir_mask, v_desc2_shift);
+	descs[2] = vbslq_u32(v_desc2_mask, v_zeros, vreinterpretq_u32_u64(descs[2]));
+
+	uint32x4_t v_desc1_shift = v_fdir_id_mask;
+	uint32x4_t v_desc1_mask = vandq_u32(v_desc_fdir_mask, v_desc1_shift);
+	descs[1] = vbslq_u32(v_desc1_mask, v_zeros, vreinterpretq_u32_u64(descs[1]));
+
+	uint32x4_t v_desc0_shift = vextq_u32(v_zeros, v_fdir_id_mask, 3);
+	uint32x4_t v_desc0_mask = vandq_u32(v_desc_fdir_mask, v_desc0_shift);
+	descs[0] = vbslq_u32(v_desc0_mask, v_zeros, vreinterpretq_u32_u64(descs[0]));
+
+	/* Shift to 1 or 0 bit per u32 lane, then to RTE_MBUF_F_RX_FDIR_ID offset */
+	RTE_BUILD_BUG_ON(RTE_MBUF_F_RX_FDIR_ID != (1 << 13));
+	uint32x4_t v_mask_one_bit = vshrq_n_u32(v_fdir_id_mask, 31);
+	return vshlq_n_u32(v_mask_one_bit, 13);
+}
+#endif
+
 static inline void
-desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
-		  struct rte_mbuf **rx_pkts)
+desc_to_olflags_v(struct i40e_rx_queue *rxq, volatile union i40e_rx_desc *rxdp,
+		  uint64x2_t descs[4], struct rte_mbuf **rx_pkts)
 {
 	uint32x4_t vlan0, vlan1, rss, l3_l4e;
 	const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0};
@@ -142,9 +263,9 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
 	vlan0 = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags,
 						vreinterpretq_u8_u32(vlan1)));
 
-	rss = vshrq_n_u32(vlan1, 11);
+	const uint32x4_t desc_fltstat = vshrq_n_u32(vlan1, 11);
 	rss = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags,
-					      vreinterpretq_u8_u32(rss)));
+					      vreinterpretq_u8_u32(desc_fltstat)));
 
 	l3_l4e = vshrq_n_u32(vlan1, 22);
 	l3_l4e = vreinterpretq_u32_u8(vqtbl1q_u8(l3_l4e_flags,
@@ -157,6 +278,18 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
 	vlan0 = vorrq_u32(vlan0, rss);
 	vlan0 = vorrq_u32(vlan0, l3_l4e);
 
+	/* Extract FDIR ID only if FDIR is enabled to avoid useless work */
+	if (rxq->fdir_enabled) {
+#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
+		uint32x4_t v_fdir_ol_flags = descs_to_fdir_32b(rxdp, rx_pkts);
+#else
+		(void)rxdp; /* rxdp not required for 16B desc mode */
+		uint32x4_t v_fdir_ol_flags = descs_to_fdir_16b(desc_fltstat, descs, rx_pkts);
+#endif
+		/* OR in ol_flag bits after descriptor specific extraction */
+		vlan0 = vorrq_u32(vlan0, v_fdir_ol_flags);
+	}
+
 	rearm0 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 0), mbuf_init, 1);
 	rearm1 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 1), mbuf_init, 1);
 	rearm2 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 2), mbuf_init, 1);
@@ -335,6 +468,8 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
 				 vreinterpretq_u16_u64(descs[0]),
 				 7));
 
+		desc_to_olflags_v(rxq, rxdp, descs, &rx_pkts[pos]);
+
 		/* D.1 pkts convert format from desc to pktmbuf */
 		pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk);
 		pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk);
@@ -363,8 +498,6 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
 
 		desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
 
-		desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
-
 		if (likely(pos + RTE_I40E_DESCS_PER_LOOP < nb_pkts)) {
 			rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP);
 		}
-- 
2.25.1

RE: [PATCH v1] net/i40e: add flow mark capability to NEON vector routine

[Zhang, Qi Z]

> -----Original Message-----
> From: Joyce Kong <joyce.kong@arm.com>
> Sent: Friday, December 17, 2021 1:36 PM
> To: Ruifeng Wang <ruifeng.wang@arm.com>; Xing, Beilei
> <beilei.xing@intel.com>
> Cc: dev@dpdk.org; nd@arm.com; Joyce Kong <joyce.kong@arm.com>
> Subject: [PATCH v1] net/i40e: add flow mark capability to NEON vector routine
> 
> This commit adds a flow director support to i40e NEON vector RX path.
> 
> I40e can have 16 and 32 byte descriptors, and the Flow Director ID data and
> indication-bit are in different locations for each size descriptor. The support is
> implemented in two separate functions as they require vastly different
> operations.
> 
> The 16B descriptor re-purposes the "filter-status" u32 field to indicate FDIR ID
> when the FLM bit is set. No extra loads are required, however we do have to
> store to mbuf->fdir.hi, which is not stored to in the RX path before this patch.
> 
> The 32B descriptor requires loading the 2nd 16 bytes of each descriptor, to get
> the FLEXBH_STAT and FD Filter ID from qword3. The resulting data must also be
> stored to
> mbuf->fdir.hi, same as the 16B code path.
> 
> Signed-off-by: Joyce Kong <joyce.kong@arm.com>
> Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>

Applied to dpdk-next-net-intel.

Thanks
Qi