From: Joyce Kong <joyce.kong@arm.com>
To: Ruifeng Wang <ruifeng.wang@arm.com>, Beilei Xing <beilei.xing@intel.com>
Cc: dev@dpdk.org, nd@arm.com, Joyce Kong <joyce.kong@arm.com>
Subject: [PATCH v1] net/i40e: add flow mark capability to NEON vector routine
Date: Fri, 17 Dec 2021 05:36:00 +0000 [thread overview]
Message-ID: <20211217053601.776086-1-joyce.kong@arm.com> (raw)
This commit adds a flow director support to i40e NEON
vector RX path.
I40e can have 16 and 32 byte descriptors, and the Flow
Director ID data and indication-bit are in different
locations for each size descriptor. The support is
implemented in two separate functions as they require
vastly different operations.
The 16B descriptor re-purposes the "filter-status" u32
field to indicate FDIR ID when the FLM bit is set. No
extra loads are required, however we do have to store
to mbuf->fdir.hi, which is not stored to in the RX path
before this patch.
The 32B descriptor requires loading the 2nd 16 bytes of
each descriptor, to get the FLEXBH_STAT and FD Filter ID
from qword3. The resulting data must also be stored to
mbuf->fdir.hi, same as the 16B code path.
Signed-off-by: Joyce Kong <joyce.kong@arm.com>
Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
---
drivers/net/i40e/i40e_rxtx_vec_neon.c | 145 ++++++++++++++++++++++++--
1 file changed, 139 insertions(+), 6 deletions(-)
diff --git a/drivers/net/i40e/i40e_rxtx_vec_neon.c b/drivers/net/i40e/i40e_rxtx_vec_neon.c
index b951ea2dc3..a7ca2c18e5 100644
--- a/drivers/net/i40e/i40e_rxtx_vec_neon.c
+++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c
@@ -77,9 +77,130 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
I40E_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rx_id);
}
+#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
+/* NEON version of FDIR mark extraction for 4 32B descriptors at a time */
+static inline uint32x4_t
+descs_to_fdir_32b(volatile union i40e_rx_desc *rxdp, struct rte_mbuf **rx_pkt)
+{
+ /* 32B descriptors: Load 2nd half of descriptors for FDIR ID data */
+ uint64x2_t desc0_qw23, desc1_qw23, desc2_qw23, desc3_qw23;
+ desc0_qw23 = vld1q_u64((uint64_t *)&(rxdp + 0)->wb.qword2);
+ desc1_qw23 = vld1q_u64((uint64_t *)&(rxdp + 1)->wb.qword2);
+ desc2_qw23 = vld1q_u64((uint64_t *)&(rxdp + 2)->wb.qword2);
+ desc3_qw23 = vld1q_u64((uint64_t *)&(rxdp + 3)->wb.qword2);
+
+ /* FDIR ID data: move last u32 of each desc to 4 u32 lanes */
+ uint32x4_t v_unpack_02, v_unpack_13;
+ v_unpack_02 = vzipq_u32(vreinterpretq_u32_u64(desc0_qw23),
+ vreinterpretq_u32_u64(desc2_qw23)).val[1];
+ v_unpack_13 = vzipq_u32(vreinterpretq_u32_u64(desc1_qw23),
+ vreinterpretq_u32_u64(desc3_qw23)).val[1];
+ uint32x4_t v_fdir_ids = vzipq_u32(v_unpack_02, v_unpack_13).val[1];
+
+ /* Extended Status: extract from each lower 32 bits, to u32 lanes */
+ v_unpack_02 = vzipq_u32(vreinterpretq_u32_u64(desc0_qw23),
+ vreinterpretq_u32_u64(desc2_qw23)).val[0];
+ v_unpack_13 = vzipq_u32(vreinterpretq_u32_u64(desc1_qw23),
+ vreinterpretq_u32_u64(desc3_qw23)).val[0];
+ uint32x4_t v_flt_status = vzipq_u32(v_unpack_02, v_unpack_13).val[0];
+
+ /* Shift u32 left and right to "mask away" bits not required.
+ * Data required is 4:5 (zero based), so left shift by 26 (32-6)
+ * and then right shift by 30 (32 - 2 bits required).
+ */
+ v_flt_status = vshlq_n_u32(v_flt_status, 26);
+ v_flt_status = vshrq_n_u32(v_flt_status, 30);
+
+ /* Generate constant 1 in all u32 lanes */
+ RTE_BUILD_BUG_ON(I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID != 1);
+ uint32x4_t v_u32_one = vdupq_n_u32(1);
+
+ /* Per desc mask, bits set if FDIR ID is valid */
+ uint32x4_t v_fd_id_mask = vceqq_u32(v_flt_status, v_u32_one);
+
+ /* Mask ID data to zero if the FD_ID bit not set in desc */
+ v_fdir_ids = vandq_u32(v_fdir_ids, v_fd_id_mask);
+
+ /* Store data to fdir.hi in mbuf */
+ rx_pkt[0]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 0);
+ rx_pkt[1]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 1);
+ rx_pkt[2]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 2);
+ rx_pkt[3]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 3);
+
+ /* Convert fdir_id_mask into a single bit, then shift as required for
+ * correct location in the mbuf->olflags
+ */
+ RTE_BUILD_BUG_ON(RTE_MBUF_F_RX_FDIR_ID != (1 << 13));
+ v_fd_id_mask = vshrq_n_u32(v_fd_id_mask, 31);
+ v_fd_id_mask = vshlq_n_u32(v_fd_id_mask, 13);
+
+ /* The returned value must be combined into each mbuf. This is already
+ * being done for RSS and VLAN mbuf olflags, so return bits to OR in.
+ */
+ return v_fd_id_mask;
+}
+
+#else /* 32 or 16B FDIR ID handling */
+
+/* Handle 16B descriptor FDIR ID flag setting based on FLM(bit11). See scalar driver
+ * for scalar implementation of the same functionality.
+ */
+static inline uint32x4_t
+descs_to_fdir_16b(uint32x4_t fltstat, uint64x2_t descs[4], struct rte_mbuf **rx_pkt)
+{
+ /* Unpack filter-status data from descriptors */
+ uint32x4_t v_tmp_02 = vzipq_u32(vreinterpretq_u32_u64(descs[0]),
+ vreinterpretq_u32_u64(descs[2])).val[0];
+ uint32x4_t v_tmp_13 = vzipq_u32(vreinterpretq_u32_u64(descs[1]),
+ vreinterpretq_u32_u64(descs[3])).val[0];
+ uint32x4_t v_fdir_ids = vzipq_u32(v_tmp_02, v_tmp_13).val[1];
+
+ /* Generate 111 and 11 in each u32 lane */
+ uint32x4_t v_111_mask = vdupq_n_u32(7);
+ uint32x4_t v_11_mask = vdupq_n_u32(3);
+
+ /* Compare and mask away FDIR ID data if bit not set */
+ uint32x4_t v_u32_bits = vandq_u32(v_111_mask, fltstat);
+ uint32x4_t v_fdir_id_mask = vceqq_u32(v_u32_bits, v_11_mask);
+ v_fdir_ids = vandq_u32(v_fdir_id_mask, v_fdir_ids);
+
+ /* Store data to fdir.hi in mbuf */
+ rx_pkt[0]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 0);
+ rx_pkt[1]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 1);
+ rx_pkt[2]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 2);
+ rx_pkt[3]->hash.fdir.hi = vgetq_lane_u32(v_fdir_ids, 3);
+
+ /* Top lane ones mask for FDIR isolation */
+ uint32x4_t v_desc_fdir_mask = {0, UINT32_MAX, 0, 0};
+
+ /* Move fdir_id_mask to correct lane, zero RSS in mbuf if fdir hits */
+ uint32x4_t v_zeros = {0, 0, 0, 0};
+ uint32x4_t v_desc3_shift = vextq_u32(v_fdir_id_mask, v_zeros, 2);
+ uint32x4_t v_desc3_mask = vandq_u32(v_desc_fdir_mask, v_desc3_shift);
+ descs[3] = vbslq_u32(v_desc3_mask, v_zeros, vreinterpretq_u32_u64(descs[3]));
+
+ uint32x4_t v_desc2_shift = vextq_u32(v_fdir_id_mask, v_zeros, 1);
+ uint32x4_t v_desc2_mask = vandq_u32(v_desc_fdir_mask, v_desc2_shift);
+ descs[2] = vbslq_u32(v_desc2_mask, v_zeros, vreinterpretq_u32_u64(descs[2]));
+
+ uint32x4_t v_desc1_shift = v_fdir_id_mask;
+ uint32x4_t v_desc1_mask = vandq_u32(v_desc_fdir_mask, v_desc1_shift);
+ descs[1] = vbslq_u32(v_desc1_mask, v_zeros, vreinterpretq_u32_u64(descs[1]));
+
+ uint32x4_t v_desc0_shift = vextq_u32(v_zeros, v_fdir_id_mask, 3);
+ uint32x4_t v_desc0_mask = vandq_u32(v_desc_fdir_mask, v_desc0_shift);
+ descs[0] = vbslq_u32(v_desc0_mask, v_zeros, vreinterpretq_u32_u64(descs[0]));
+
+ /* Shift to 1 or 0 bit per u32 lane, then to RTE_MBUF_F_RX_FDIR_ID offset */
+ RTE_BUILD_BUG_ON(RTE_MBUF_F_RX_FDIR_ID != (1 << 13));
+ uint32x4_t v_mask_one_bit = vshrq_n_u32(v_fdir_id_mask, 31);
+ return vshlq_n_u32(v_mask_one_bit, 13);
+}
+#endif
+
static inline void
-desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
- struct rte_mbuf **rx_pkts)
+desc_to_olflags_v(struct i40e_rx_queue *rxq, volatile union i40e_rx_desc *rxdp,
+ uint64x2_t descs[4], struct rte_mbuf **rx_pkts)
{
uint32x4_t vlan0, vlan1, rss, l3_l4e;
const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0};
@@ -142,9 +263,9 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
vlan0 = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags,
vreinterpretq_u8_u32(vlan1)));
- rss = vshrq_n_u32(vlan1, 11);
+ const uint32x4_t desc_fltstat = vshrq_n_u32(vlan1, 11);
rss = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags,
- vreinterpretq_u8_u32(rss)));
+ vreinterpretq_u8_u32(desc_fltstat)));
l3_l4e = vshrq_n_u32(vlan1, 22);
l3_l4e = vreinterpretq_u32_u8(vqtbl1q_u8(l3_l4e_flags,
@@ -157,6 +278,18 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
vlan0 = vorrq_u32(vlan0, rss);
vlan0 = vorrq_u32(vlan0, l3_l4e);
+ /* Extract FDIR ID only if FDIR is enabled to avoid useless work */
+ if (rxq->fdir_enabled) {
+#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
+ uint32x4_t v_fdir_ol_flags = descs_to_fdir_32b(rxdp, rx_pkts);
+#else
+ (void)rxdp; /* rxdp not required for 16B desc mode */
+ uint32x4_t v_fdir_ol_flags = descs_to_fdir_16b(desc_fltstat, descs, rx_pkts);
+#endif
+ /* OR in ol_flag bits after descriptor specific extraction */
+ vlan0 = vorrq_u32(vlan0, v_fdir_ol_flags);
+ }
+
rearm0 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 0), mbuf_init, 1);
rearm1 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 1), mbuf_init, 1);
rearm2 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 2), mbuf_init, 1);
@@ -335,6 +468,8 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
vreinterpretq_u16_u64(descs[0]),
7));
+ desc_to_olflags_v(rxq, rxdp, descs, &rx_pkts[pos]);
+
/* D.1 pkts convert format from desc to pktmbuf */
pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk);
pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk);
@@ -363,8 +498,6 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *__rte_restrict rxq,
desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
- desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
-
if (likely(pos + RTE_I40E_DESCS_PER_LOOP < nb_pkts)) {
rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP);
}
--
2.25.1
next reply other threads:[~2021-12-17 5:36 UTC|newest]
Thread overview: 2+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-12-17 5:36 Joyce Kong [this message]
2022-02-15 8:03 ` Zhang, Qi Z
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211217053601.776086-1-joyce.kong@arm.com \
--to=joyce.kong@arm.com \
--cc=beilei.xing@intel.com \
--cc=dev@dpdk.org \
--cc=nd@arm.com \
--cc=ruifeng.wang@arm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).