From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id F40BBA0548; Sat, 6 Mar 2021 16:36:53 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 37AB322A391; Sat, 6 Mar 2021 16:35:09 +0100 (CET) Received: from mx0b-0016f401.pphosted.com (mx0a-0016f401.pphosted.com [67.231.148.174]) by mails.dpdk.org (Postfix) with ESMTP id 29AE722A3D3 for ; Sat, 6 Mar 2021 16:35:07 +0100 (CET) Received: from pps.filterd (m0045849.ppops.net [127.0.0.1]) by mx0a-0016f401.pphosted.com (8.16.0.43/8.16.0.43) with SMTP id 126FWmqT029543 for ; Sat, 6 Mar 2021 07:35:06 -0800 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h=from : to : cc : subject : date : message-id : in-reply-to : references : mime-version : content-type; s=pfpt0220; bh=duPs3CRvzKB/MFN3qUDup41qZhbGx3ftD2U5BD5EY9s=; b=EOoiri0JDX0L7mUt9ta7oyZPKNnrNm79oYwl6L7sFCsoCABaFwLzkmzKG32sdL7bgOZQ rv75Qdn/yaAGxQVGhULN3iXeEaAnjCoZuUW/g6VYBgtHMFM0AfaOJ/UGqmmYYv3Fnvue PqTMsgFn0krkKTMSDDSkdEP1F9SGrEirkdnGo8tYvWKwpjSZ8HRMeBV6mOPqIFKyPFMj 9zNaWCbi91lIi6m/O8xtS/Jz3qOqB82tFij3UymIykIFResZemIhp2p3juMhdMz+xWMG dLFmyEqOXAjypN8aRBXSfAAafE1xS2qpnr093FLLDNzuRirmQCE9AVaE1TymZHHXzOZU NA== Received: from dc5-exch01.marvell.com ([199.233.59.181]) by mx0a-0016f401.pphosted.com with ESMTP id 3747yurcc4-2 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT) for ; Sat, 06 Mar 2021 07:35:06 -0800 Received: from DC5-EXCH01.marvell.com (10.69.176.38) by DC5-EXCH01.marvell.com (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.2; Sat, 6 Mar 2021 07:35:04 -0800 Received: from maili.marvell.com (10.69.176.80) by DC5-EXCH01.marvell.com (10.69.176.38) with Microsoft SMTP Server id 15.0.1497.2 via Frontend Transport; Sat, 6 Mar 2021 07:35:04 -0800 Received: from hyd1588t430.marvell.com (unknown [10.29.52.204]) by maili.marvell.com (Postfix) with ESMTP id 969313F703F; Sat, 6 Mar 2021 07:35:01 -0800 (PST) From: Nithin Dabilpuram To: CC: , , , , , , , Nithin Dabilpuram Date: Sat, 6 Mar 2021 21:03:36 +0530 Message-ID: <20210306153404.10781-17-ndabilpuram@marvell.com> X-Mailer: git-send-email 2.8.4 In-Reply-To: <20210306153404.10781-1-ndabilpuram@marvell.com> References: <20210306153404.10781-1-ndabilpuram@marvell.com> MIME-Version: 1.0 Content-Type: text/plain X-Proofpoint-Virus-Version: vendor=fsecure engine=2.50.10434:6.0.369, 18.0.761 definitions=2021-03-06_08:2021-03-03, 2021-03-06 signatures=0 Subject: [dpdk-dev] [PATCH 16/44] net/cnxk: add Tx vector version for cn9k X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add Tx burst vector version for CN9K. Signed-off-by: Nithin Dabilpuram Signed-off-by: Pavan Nikhilesh --- drivers/net/cnxk/cn9k_tx.c | 951 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 950 insertions(+), 1 deletion(-) diff --git a/drivers/net/cnxk/cn9k_tx.c b/drivers/net/cnxk/cn9k_tx.c index a474eb5..300ccd2 100644 --- a/drivers/net/cnxk/cn9k_tx.c +++ b/drivers/net/cnxk/cn9k_tx.c @@ -2,6 +2,8 @@ * Copyright(C) 2021 Marvell. */ +#include + #include "cn9k_ethdev.h" #include "cn9k_tx.h" @@ -93,6 +95,921 @@ nix_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts, return pkts; } +#if defined(RTE_ARCH_ARM64) + +#define NIX_DESCS_PER_LOOP 4 +static __rte_always_inline uint16_t +nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts, + uint64_t *cmd, const uint16_t flags) +{ + uint64x2_t dataoff_iova0, dataoff_iova1, dataoff_iova2, dataoff_iova3; + uint64x2_t len_olflags0, len_olflags1, len_olflags2, len_olflags3; + uint64_t *mbuf0, *mbuf1, *mbuf2, *mbuf3; + uint64x2_t senddesc01_w0, senddesc23_w0; + uint64x2_t senddesc01_w1, senddesc23_w1; + uint64x2_t sgdesc01_w0, sgdesc23_w0; + uint64x2_t sgdesc01_w1, sgdesc23_w1; + struct cn9k_eth_txq *txq = tx_queue; + uint64_t *lmt_addr = txq->lmt_addr; + rte_iova_t io_addr = txq->io_addr; + uint64x2_t ltypes01, ltypes23; + uint64x2_t xtmp128, ytmp128; + uint64x2_t xmask01, xmask23; + uint64x2_t cmd00, cmd01; + uint64x2_t cmd10, cmd11; + uint64x2_t cmd20, cmd21; + uint64x2_t cmd30, cmd31; + uint64_t lmt_status, i; + uint16_t pkts_left; + + NIX_XMIT_FC_OR_RETURN(txq, pkts); + + pkts_left = pkts & (NIX_DESCS_PER_LOOP - 1); + pkts = RTE_ALIGN_FLOOR(pkts, NIX_DESCS_PER_LOOP); + + /* Reduce the cached count */ + txq->fc_cache_pkts -= pkts; + + /* Lets commit any changes in the packet here as no further changes + * to the packet will be done unless no fast free is enabled. + */ + if (!(flags & NIX_TX_OFFLOAD_MBUF_NOFF_F)) + rte_io_wmb(); + + senddesc01_w0 = vld1q_dup_u64(&txq->cmd[0]); + senddesc23_w0 = senddesc01_w0; + senddesc01_w1 = vdupq_n_u64(0); + senddesc23_w1 = senddesc01_w1; + sgdesc01_w0 = vld1q_dup_u64(&txq->cmd[2]); + sgdesc23_w0 = sgdesc01_w0; + + for (i = 0; i < pkts; i += NIX_DESCS_PER_LOOP) { + /* Clear lower 32bit of SEND_HDR_W0 and SEND_SG_W0 */ + senddesc01_w0 = + vbicq_u64(senddesc01_w0, vdupq_n_u64(0xFFFFFFFF)); + sgdesc01_w0 = vbicq_u64(sgdesc01_w0, vdupq_n_u64(0xFFFFFFFF)); + + senddesc23_w0 = senddesc01_w0; + sgdesc23_w0 = sgdesc01_w0; + + /* Move mbufs to iova */ + mbuf0 = (uint64_t *)tx_pkts[0]; + mbuf1 = (uint64_t *)tx_pkts[1]; + mbuf2 = (uint64_t *)tx_pkts[2]; + mbuf3 = (uint64_t *)tx_pkts[3]; + + mbuf0 = (uint64_t *)((uintptr_t)mbuf0 + + offsetof(struct rte_mbuf, buf_iova)); + mbuf1 = (uint64_t *)((uintptr_t)mbuf1 + + offsetof(struct rte_mbuf, buf_iova)); + mbuf2 = (uint64_t *)((uintptr_t)mbuf2 + + offsetof(struct rte_mbuf, buf_iova)); + mbuf3 = (uint64_t *)((uintptr_t)mbuf3 + + offsetof(struct rte_mbuf, buf_iova)); + /* + * Get mbuf's, olflags, iova, pktlen, dataoff + * dataoff_iovaX.D[0] = iova, + * dataoff_iovaX.D[1](15:0) = mbuf->dataoff + * len_olflagsX.D[0] = ol_flags, + * len_olflagsX.D[1](63:32) = mbuf->pkt_len + */ + dataoff_iova0 = vld1q_u64(mbuf0); + len_olflags0 = vld1q_u64(mbuf0 + 2); + dataoff_iova1 = vld1q_u64(mbuf1); + len_olflags1 = vld1q_u64(mbuf1 + 2); + dataoff_iova2 = vld1q_u64(mbuf2); + len_olflags2 = vld1q_u64(mbuf2 + 2); + dataoff_iova3 = vld1q_u64(mbuf3); + len_olflags3 = vld1q_u64(mbuf3 + 2); + + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { + struct rte_mbuf *mbuf; + /* Set don't free bit if reference count > 1 */ + xmask01 = vdupq_n_u64(0); + xmask23 = xmask01; + + mbuf = (struct rte_mbuf *)((uintptr_t)mbuf0 - + offsetof(struct rte_mbuf, + buf_iova)); + + if (cnxk_nix_prefree_seg(mbuf)) + vsetq_lane_u64(0x80000, xmask01, 0); + else + __mempool_check_cookies(mbuf->pool, + (void **)&mbuf, 1, 0); + + mbuf = (struct rte_mbuf *)((uintptr_t)mbuf1 - + offsetof(struct rte_mbuf, + buf_iova)); + if (cnxk_nix_prefree_seg(mbuf)) + vsetq_lane_u64(0x80000, xmask01, 1); + else + __mempool_check_cookies(mbuf->pool, + (void **)&mbuf, 1, 0); + + mbuf = (struct rte_mbuf *)((uintptr_t)mbuf2 - + offsetof(struct rte_mbuf, + buf_iova)); + if (cnxk_nix_prefree_seg(mbuf)) + vsetq_lane_u64(0x80000, xmask23, 0); + else + __mempool_check_cookies(mbuf->pool, + (void **)&mbuf, 1, 0); + + mbuf = (struct rte_mbuf *)((uintptr_t)mbuf3 - + offsetof(struct rte_mbuf, + buf_iova)); + if (cnxk_nix_prefree_seg(mbuf)) + vsetq_lane_u64(0x80000, xmask23, 1); + else + __mempool_check_cookies(mbuf->pool, + (void **)&mbuf, 1, 0); + senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01); + senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23); + /* Ensuring mbuf fields which got updated in + * cnxk_nix_prefree_seg are written before LMTST. + */ + rte_io_wmb(); + } else { + struct rte_mbuf *mbuf; + /* Mark mempool object as "put" since + * it is freed by NIX + */ + mbuf = (struct rte_mbuf *)((uintptr_t)mbuf0 - + offsetof(struct rte_mbuf, + buf_iova)); + __mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, + 0); + + mbuf = (struct rte_mbuf *)((uintptr_t)mbuf1 - + offsetof(struct rte_mbuf, + buf_iova)); + __mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, + 0); + + mbuf = (struct rte_mbuf *)((uintptr_t)mbuf2 - + offsetof(struct rte_mbuf, + buf_iova)); + __mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, + 0); + + mbuf = (struct rte_mbuf *)((uintptr_t)mbuf3 - + offsetof(struct rte_mbuf, + buf_iova)); + __mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, + 0); + RTE_SET_USED(mbuf); + } + + /* Move mbufs to point pool */ + mbuf0 = (uint64_t *)((uintptr_t)mbuf0 + + offsetof(struct rte_mbuf, pool) - + offsetof(struct rte_mbuf, buf_iova)); + mbuf1 = (uint64_t *)((uintptr_t)mbuf1 + + offsetof(struct rte_mbuf, pool) - + offsetof(struct rte_mbuf, buf_iova)); + mbuf2 = (uint64_t *)((uintptr_t)mbuf2 + + offsetof(struct rte_mbuf, pool) - + offsetof(struct rte_mbuf, buf_iova)); + mbuf3 = (uint64_t *)((uintptr_t)mbuf3 + + offsetof(struct rte_mbuf, pool) - + offsetof(struct rte_mbuf, buf_iova)); + + if (flags & (NIX_TX_OFFLOAD_OL3_OL4_CSUM_F | + NIX_TX_OFFLOAD_L3_L4_CSUM_F)) { + /* Get tx_offload for ol2, ol3, l2, l3 lengths */ + /* + * E(8):OL2_LEN(7):OL3_LEN(9):E(24):L3_LEN(9):L2_LEN(7) + * E(8):OL2_LEN(7):OL3_LEN(9):E(24):L3_LEN(9):L2_LEN(7) + */ + + asm volatile("LD1 {%[a].D}[0],[%[in]]\n\t" + : [a] "+w"(senddesc01_w1) + : [in] "r"(mbuf0 + 2) + : "memory"); + + asm volatile("LD1 {%[a].D}[1],[%[in]]\n\t" + : [a] "+w"(senddesc01_w1) + : [in] "r"(mbuf1 + 2) + : "memory"); + + asm volatile("LD1 {%[b].D}[0],[%[in]]\n\t" + : [b] "+w"(senddesc23_w1) + : [in] "r"(mbuf2 + 2) + : "memory"); + + asm volatile("LD1 {%[b].D}[1],[%[in]]\n\t" + : [b] "+w"(senddesc23_w1) + : [in] "r"(mbuf3 + 2) + : "memory"); + + /* Get pool pointer alone */ + mbuf0 = (uint64_t *)*mbuf0; + mbuf1 = (uint64_t *)*mbuf1; + mbuf2 = (uint64_t *)*mbuf2; + mbuf3 = (uint64_t *)*mbuf3; + } else { + /* Get pool pointer alone */ + mbuf0 = (uint64_t *)*mbuf0; + mbuf1 = (uint64_t *)*mbuf1; + mbuf2 = (uint64_t *)*mbuf2; + mbuf3 = (uint64_t *)*mbuf3; + } + + const uint8x16_t shuf_mask2 = { + 0x4, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + }; + xtmp128 = vzip2q_u64(len_olflags0, len_olflags1); + ytmp128 = vzip2q_u64(len_olflags2, len_olflags3); + + /* Clear dataoff_iovaX.D[1] bits other than dataoff(15:0) */ + const uint64x2_t and_mask0 = { + 0xFFFFFFFFFFFFFFFF, + 0x000000000000FFFF, + }; + + dataoff_iova0 = vandq_u64(dataoff_iova0, and_mask0); + dataoff_iova1 = vandq_u64(dataoff_iova1, and_mask0); + dataoff_iova2 = vandq_u64(dataoff_iova2, and_mask0); + dataoff_iova3 = vandq_u64(dataoff_iova3, and_mask0); + + /* + * Pick only 16 bits of pktlen preset at bits 63:32 + * and place them at bits 15:0. + */ + xtmp128 = vqtbl1q_u8(xtmp128, shuf_mask2); + ytmp128 = vqtbl1q_u8(ytmp128, shuf_mask2); + + /* Add pairwise to get dataoff + iova in sgdesc_w1 */ + sgdesc01_w1 = vpaddq_u64(dataoff_iova0, dataoff_iova1); + sgdesc23_w1 = vpaddq_u64(dataoff_iova2, dataoff_iova3); + + /* Orr both sgdesc_w0 and senddesc_w0 with 16 bits of + * pktlen at 15:0 position. + */ + sgdesc01_w0 = vorrq_u64(sgdesc01_w0, xtmp128); + sgdesc23_w0 = vorrq_u64(sgdesc23_w0, ytmp128); + senddesc01_w0 = vorrq_u64(senddesc01_w0, xtmp128); + senddesc23_w0 = vorrq_u64(senddesc23_w0, ytmp128); + + if ((flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) && + !(flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) { + /* + * Lookup table to translate ol_flags to + * il3/il4 types. But we still use ol3/ol4 types in + * senddesc_w1 as only one header processing is enabled. + */ + const uint8x16_t tbl = { + /* [0-15] = il4type:il3type */ + 0x04, /* none (IPv6 assumed) */ + 0x14, /* PKT_TX_TCP_CKSUM (IPv6 assumed) */ + 0x24, /* PKT_TX_SCTP_CKSUM (IPv6 assumed) */ + 0x34, /* PKT_TX_UDP_CKSUM (IPv6 assumed) */ + 0x03, /* PKT_TX_IP_CKSUM */ + 0x13, /* PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM */ + 0x23, /* PKT_TX_IP_CKSUM | PKT_TX_SCTP_CKSUM */ + 0x33, /* PKT_TX_IP_CKSUM | PKT_TX_UDP_CKSUM */ + 0x02, /* PKT_TX_IPV4 */ + 0x12, /* PKT_TX_IPV4 | PKT_TX_TCP_CKSUM */ + 0x22, /* PKT_TX_IPV4 | PKT_TX_SCTP_CKSUM */ + 0x32, /* PKT_TX_IPV4 | PKT_TX_UDP_CKSUM */ + 0x03, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM */ + 0x13, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM | + * PKT_TX_TCP_CKSUM + */ + 0x23, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM | + * PKT_TX_SCTP_CKSUM + */ + 0x33, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM | + * PKT_TX_UDP_CKSUM + */ + }; + + /* Extract olflags to translate to iltypes */ + xtmp128 = vzip1q_u64(len_olflags0, len_olflags1); + ytmp128 = vzip1q_u64(len_olflags2, len_olflags3); + + /* + * E(47):L3_LEN(9):L2_LEN(7+z) + * E(47):L3_LEN(9):L2_LEN(7+z) + */ + senddesc01_w1 = vshlq_n_u64(senddesc01_w1, 1); + senddesc23_w1 = vshlq_n_u64(senddesc23_w1, 1); + + /* Move OLFLAGS bits 55:52 to 51:48 + * with zeros preprended on the byte and rest + * don't care + */ + xtmp128 = vshrq_n_u8(xtmp128, 4); + ytmp128 = vshrq_n_u8(ytmp128, 4); + /* + * E(48):L3_LEN(8):L2_LEN(z+7) + * E(48):L3_LEN(8):L2_LEN(z+7) + */ + const int8x16_t tshft3 = { + -1, 0, 8, 8, 8, 8, 8, 8, + -1, 0, 8, 8, 8, 8, 8, 8, + }; + + senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3); + senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3); + + /* Do the lookup */ + ltypes01 = vqtbl1q_u8(tbl, xtmp128); + ltypes23 = vqtbl1q_u8(tbl, ytmp128); + + /* Just use ld1q to retrieve aura + * when we don't need tx_offload + */ + mbuf0 = (uint64_t *)((uintptr_t)mbuf0 + + offsetof(struct rte_mempool, + pool_id)); + mbuf1 = (uint64_t *)((uintptr_t)mbuf1 + + offsetof(struct rte_mempool, + pool_id)); + mbuf2 = (uint64_t *)((uintptr_t)mbuf2 + + offsetof(struct rte_mempool, + pool_id)); + mbuf3 = (uint64_t *)((uintptr_t)mbuf3 + + offsetof(struct rte_mempool, + pool_id)); + + /* Pick only relevant fields i.e Bit 48:55 of iltype + * and place it in ol3/ol4type of senddesc_w1 + */ + const uint8x16_t shuf_mask0 = { + 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xE, 0xFF, 0xFF, 0xFF, + }; + + ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0); + ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0); + + /* Prepare ol4ptr, ol3ptr from ol3len, ol2len. + * a [E(32):E(16):OL3(8):OL2(8)] + * a = a + (a << 8) + * a [E(32):E(16):(OL3+OL2):OL2] + * => E(32):E(16)::OL4PTR(8):OL3PTR(8) + */ + senddesc01_w1 = vaddq_u8(senddesc01_w1, + vshlq_n_u16(senddesc01_w1, 8)); + senddesc23_w1 = vaddq_u8(senddesc23_w1, + vshlq_n_u16(senddesc23_w1, 8)); + + /* Create first half of 4W cmd for 4 mbufs (sgdesc) */ + cmd01 = vzip1q_u64(sgdesc01_w0, sgdesc01_w1); + cmd11 = vzip2q_u64(sgdesc01_w0, sgdesc01_w1); + cmd21 = vzip1q_u64(sgdesc23_w0, sgdesc23_w1); + cmd31 = vzip2q_u64(sgdesc23_w0, sgdesc23_w1); + + xmask01 = vdupq_n_u64(0); + xmask23 = xmask01; + asm volatile("LD1 {%[a].H}[0],[%[in]]\n\t" + : [a] "+w"(xmask01) + : [in] "r"(mbuf0) + : "memory"); + + asm volatile("LD1 {%[a].H}[4],[%[in]]\n\t" + : [a] "+w"(xmask01) + : [in] "r"(mbuf1) + : "memory"); + + asm volatile("LD1 {%[b].H}[0],[%[in]]\n\t" + : [b] "+w"(xmask23) + : [in] "r"(mbuf2) + : "memory"); + + asm volatile("LD1 {%[b].H}[4],[%[in]]\n\t" + : [b] "+w"(xmask23) + : [in] "r"(mbuf3) + : "memory"); + xmask01 = vshlq_n_u64(xmask01, 20); + xmask23 = vshlq_n_u64(xmask23, 20); + + senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01); + senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23); + /* Move ltypes to senddesc*_w1 */ + senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01); + senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23); + + /* Create first half of 4W cmd for 4 mbufs (sendhdr) */ + cmd00 = vzip1q_u64(senddesc01_w0, senddesc01_w1); + cmd10 = vzip2q_u64(senddesc01_w0, senddesc01_w1); + cmd20 = vzip1q_u64(senddesc23_w0, senddesc23_w1); + cmd30 = vzip2q_u64(senddesc23_w0, senddesc23_w1); + + } else if (!(flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) && + (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) { + /* + * Lookup table to translate ol_flags to + * ol3/ol4 types. + */ + + const uint8x16_t tbl = { + /* [0-15] = ol4type:ol3type */ + 0x00, /* none */ + 0x03, /* OUTER_IP_CKSUM */ + 0x02, /* OUTER_IPV4 */ + 0x03, /* OUTER_IPV4 | OUTER_IP_CKSUM */ + 0x04, /* OUTER_IPV6 */ + 0x00, /* OUTER_IPV6 | OUTER_IP_CKSUM */ + 0x00, /* OUTER_IPV6 | OUTER_IPV4 */ + 0x00, /* OUTER_IPV6 | OUTER_IPV4 | + * OUTER_IP_CKSUM + */ + 0x00, /* OUTER_UDP_CKSUM */ + 0x33, /* OUTER_UDP_CKSUM | OUTER_IP_CKSUM */ + 0x32, /* OUTER_UDP_CKSUM | OUTER_IPV4 */ + 0x33, /* OUTER_UDP_CKSUM | OUTER_IPV4 | + * OUTER_IP_CKSUM + */ + 0x34, /* OUTER_UDP_CKSUM | OUTER_IPV6 */ + 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 | + * OUTER_IP_CKSUM + */ + 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 | + * OUTER_IPV4 + */ + 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 | + * OUTER_IPV4 | OUTER_IP_CKSUM + */ + }; + + /* Extract olflags to translate to iltypes */ + xtmp128 = vzip1q_u64(len_olflags0, len_olflags1); + ytmp128 = vzip1q_u64(len_olflags2, len_olflags3); + + /* + * E(47):OL3_LEN(9):OL2_LEN(7+z) + * E(47):OL3_LEN(9):OL2_LEN(7+z) + */ + const uint8x16_t shuf_mask5 = { + 0x6, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE, 0xD, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + }; + senddesc01_w1 = vqtbl1q_u8(senddesc01_w1, shuf_mask5); + senddesc23_w1 = vqtbl1q_u8(senddesc23_w1, shuf_mask5); + + /* Extract outer ol flags only */ + const uint64x2_t o_cksum_mask = { + 0x1C00020000000000, + 0x1C00020000000000, + }; + + xtmp128 = vandq_u64(xtmp128, o_cksum_mask); + ytmp128 = vandq_u64(ytmp128, o_cksum_mask); + + /* Extract OUTER_UDP_CKSUM bit 41 and + * move it to bit 61 + */ + + xtmp128 = xtmp128 | vshlq_n_u64(xtmp128, 20); + ytmp128 = ytmp128 | vshlq_n_u64(ytmp128, 20); + + /* Shift oltype by 2 to start nibble from BIT(56) + * instead of BIT(58) + */ + xtmp128 = vshrq_n_u8(xtmp128, 2); + ytmp128 = vshrq_n_u8(ytmp128, 2); + /* + * E(48):L3_LEN(8):L2_LEN(z+7) + * E(48):L3_LEN(8):L2_LEN(z+7) + */ + const int8x16_t tshft3 = { + -1, 0, 8, 8, 8, 8, 8, 8, + -1, 0, 8, 8, 8, 8, 8, 8, + }; + + senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3); + senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3); + + /* Do the lookup */ + ltypes01 = vqtbl1q_u8(tbl, xtmp128); + ltypes23 = vqtbl1q_u8(tbl, ytmp128); + + /* Just use ld1q to retrieve aura + * when we don't need tx_offload + */ + mbuf0 = (uint64_t *)((uintptr_t)mbuf0 + + offsetof(struct rte_mempool, + pool_id)); + mbuf1 = (uint64_t *)((uintptr_t)mbuf1 + + offsetof(struct rte_mempool, + pool_id)); + mbuf2 = (uint64_t *)((uintptr_t)mbuf2 + + offsetof(struct rte_mempool, + pool_id)); + mbuf3 = (uint64_t *)((uintptr_t)mbuf3 + + offsetof(struct rte_mempool, + pool_id)); + + /* Pick only relevant fields i.e Bit 56:63 of oltype + * and place it in ol3/ol4type of senddesc_w1 + */ + const uint8x16_t shuf_mask0 = { + 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xF, 0xFF, 0xFF, 0xFF, + }; + + ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0); + ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0); + + /* Prepare ol4ptr, ol3ptr from ol3len, ol2len. + * a [E(32):E(16):OL3(8):OL2(8)] + * a = a + (a << 8) + * a [E(32):E(16):(OL3+OL2):OL2] + * => E(32):E(16)::OL4PTR(8):OL3PTR(8) + */ + senddesc01_w1 = vaddq_u8(senddesc01_w1, + vshlq_n_u16(senddesc01_w1, 8)); + senddesc23_w1 = vaddq_u8(senddesc23_w1, + vshlq_n_u16(senddesc23_w1, 8)); + + /* Create second half of 4W cmd for 4 mbufs (sgdesc) */ + cmd01 = vzip1q_u64(sgdesc01_w0, sgdesc01_w1); + cmd11 = vzip2q_u64(sgdesc01_w0, sgdesc01_w1); + cmd21 = vzip1q_u64(sgdesc23_w0, sgdesc23_w1); + cmd31 = vzip2q_u64(sgdesc23_w0, sgdesc23_w1); + + xmask01 = vdupq_n_u64(0); + xmask23 = xmask01; + asm volatile("LD1 {%[a].H}[0],[%[in]]\n\t" + : [a] "+w"(xmask01) + : [in] "r"(mbuf0) + : "memory"); + + asm volatile("LD1 {%[a].H}[4],[%[in]]\n\t" + : [a] "+w"(xmask01) + : [in] "r"(mbuf1) + : "memory"); + + asm volatile("LD1 {%[b].H}[0],[%[in]]\n\t" + : [b] "+w"(xmask23) + : [in] "r"(mbuf2) + : "memory"); + + asm volatile("LD1 {%[b].H}[4],[%[in]]\n\t" + : [b] "+w"(xmask23) + : [in] "r"(mbuf3) + : "memory"); + xmask01 = vshlq_n_u64(xmask01, 20); + xmask23 = vshlq_n_u64(xmask23, 20); + + senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01); + senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23); + /* Move ltypes to senddesc*_w1 */ + senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01); + senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23); + + /* Create first half of 4W cmd for 4 mbufs (sendhdr) */ + cmd00 = vzip1q_u64(senddesc01_w0, senddesc01_w1); + cmd10 = vzip2q_u64(senddesc01_w0, senddesc01_w1); + cmd20 = vzip1q_u64(senddesc23_w0, senddesc23_w1); + cmd30 = vzip2q_u64(senddesc23_w0, senddesc23_w1); + + } else if ((flags & NIX_TX_OFFLOAD_L3_L4_CSUM_F) && + (flags & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)) { + /* Lookup table to translate ol_flags to + * ol4type, ol3type, il4type, il3type of senddesc_w1 + */ + const uint8x16x2_t tbl = {{ + { + /* [0-15] = il4type:il3type */ + 0x04, /* none (IPv6) */ + 0x14, /* PKT_TX_TCP_CKSUM (IPv6) */ + 0x24, /* PKT_TX_SCTP_CKSUM (IPv6) */ + 0x34, /* PKT_TX_UDP_CKSUM (IPv6) */ + 0x03, /* PKT_TX_IP_CKSUM */ + 0x13, /* PKT_TX_IP_CKSUM | + * PKT_TX_TCP_CKSUM + */ + 0x23, /* PKT_TX_IP_CKSUM | + * PKT_TX_SCTP_CKSUM + */ + 0x33, /* PKT_TX_IP_CKSUM | + * PKT_TX_UDP_CKSUM + */ + 0x02, /* PKT_TX_IPV4 */ + 0x12, /* PKT_TX_IPV4 | + * PKT_TX_TCP_CKSUM + */ + 0x22, /* PKT_TX_IPV4 | + * PKT_TX_SCTP_CKSUM + */ + 0x32, /* PKT_TX_IPV4 | + * PKT_TX_UDP_CKSUM + */ + 0x03, /* PKT_TX_IPV4 | + * PKT_TX_IP_CKSUM + */ + 0x13, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM | + * PKT_TX_TCP_CKSUM + */ + 0x23, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM | + * PKT_TX_SCTP_CKSUM + */ + 0x33, /* PKT_TX_IPV4 | PKT_TX_IP_CKSUM | + * PKT_TX_UDP_CKSUM + */ + }, + + { + /* [16-31] = ol4type:ol3type */ + 0x00, /* none */ + 0x03, /* OUTER_IP_CKSUM */ + 0x02, /* OUTER_IPV4 */ + 0x03, /* OUTER_IPV4 | OUTER_IP_CKSUM */ + 0x04, /* OUTER_IPV6 */ + 0x00, /* OUTER_IPV6 | OUTER_IP_CKSUM */ + 0x00, /* OUTER_IPV6 | OUTER_IPV4 */ + 0x00, /* OUTER_IPV6 | OUTER_IPV4 | + * OUTER_IP_CKSUM + */ + 0x00, /* OUTER_UDP_CKSUM */ + 0x33, /* OUTER_UDP_CKSUM | + * OUTER_IP_CKSUM + */ + 0x32, /* OUTER_UDP_CKSUM | + * OUTER_IPV4 + */ + 0x33, /* OUTER_UDP_CKSUM | + * OUTER_IPV4 | OUTER_IP_CKSUM + */ + 0x34, /* OUTER_UDP_CKSUM | + * OUTER_IPV6 + */ + 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 | + * OUTER_IP_CKSUM + */ + 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 | + * OUTER_IPV4 + */ + 0x00, /* OUTER_UDP_CKSUM | OUTER_IPV6 | + * OUTER_IPV4 | OUTER_IP_CKSUM + */ + }, + }}; + + /* Extract olflags to translate to oltype & iltype */ + xtmp128 = vzip1q_u64(len_olflags0, len_olflags1); + ytmp128 = vzip1q_u64(len_olflags2, len_olflags3); + + /* + * E(8):OL2_LN(7):OL3_LN(9):E(23):L3_LN(9):L2_LN(7+z) + * E(8):OL2_LN(7):OL3_LN(9):E(23):L3_LN(9):L2_LN(7+z) + */ + const uint32x4_t tshft_4 = { + 1, + 0, + 1, + 0, + }; + senddesc01_w1 = vshlq_u32(senddesc01_w1, tshft_4); + senddesc23_w1 = vshlq_u32(senddesc23_w1, tshft_4); + + /* + * E(32):L3_LEN(8):L2_LEN(7+Z):OL3_LEN(8):OL2_LEN(7+Z) + * E(32):L3_LEN(8):L2_LEN(7+Z):OL3_LEN(8):OL2_LEN(7+Z) + */ + const uint8x16_t shuf_mask5 = { + 0x6, 0x5, 0x0, 0x1, 0xFF, 0xFF, 0xFF, 0xFF, + 0xE, 0xD, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, + }; + senddesc01_w1 = vqtbl1q_u8(senddesc01_w1, shuf_mask5); + senddesc23_w1 = vqtbl1q_u8(senddesc23_w1, shuf_mask5); + + /* Extract outer and inner header ol_flags */ + const uint64x2_t oi_cksum_mask = { + 0x1CF0020000000000, + 0x1CF0020000000000, + }; + + xtmp128 = vandq_u64(xtmp128, oi_cksum_mask); + ytmp128 = vandq_u64(ytmp128, oi_cksum_mask); + + /* Extract OUTER_UDP_CKSUM bit 41 and + * move it to bit 61 + */ + + xtmp128 = xtmp128 | vshlq_n_u64(xtmp128, 20); + ytmp128 = ytmp128 | vshlq_n_u64(ytmp128, 20); + + /* Shift right oltype by 2 and iltype by 4 + * to start oltype nibble from BIT(58) + * instead of BIT(56) and iltype nibble from BIT(48) + * instead of BIT(52). + */ + const int8x16_t tshft5 = { + 8, 8, 8, 8, 8, 8, -4, -2, + 8, 8, 8, 8, 8, 8, -4, -2, + }; + + xtmp128 = vshlq_u8(xtmp128, tshft5); + ytmp128 = vshlq_u8(ytmp128, tshft5); + /* + * E(32):L3_LEN(8):L2_LEN(8):OL3_LEN(8):OL2_LEN(8) + * E(32):L3_LEN(8):L2_LEN(8):OL3_LEN(8):OL2_LEN(8) + */ + const int8x16_t tshft3 = { + -1, 0, -1, 0, 0, 0, 0, 0, + -1, 0, -1, 0, 0, 0, 0, 0, + }; + + senddesc01_w1 = vshlq_u8(senddesc01_w1, tshft3); + senddesc23_w1 = vshlq_u8(senddesc23_w1, tshft3); + + /* Mark Bit(4) of oltype */ + const uint64x2_t oi_cksum_mask2 = { + 0x1000000000000000, + 0x1000000000000000, + }; + + xtmp128 = vorrq_u64(xtmp128, oi_cksum_mask2); + ytmp128 = vorrq_u64(ytmp128, oi_cksum_mask2); + + /* Do the lookup */ + ltypes01 = vqtbl2q_u8(tbl, xtmp128); + ltypes23 = vqtbl2q_u8(tbl, ytmp128); + + /* Just use ld1q to retrieve aura + * when we don't need tx_offload + */ + mbuf0 = (uint64_t *)((uintptr_t)mbuf0 + + offsetof(struct rte_mempool, + pool_id)); + mbuf1 = (uint64_t *)((uintptr_t)mbuf1 + + offsetof(struct rte_mempool, + pool_id)); + mbuf2 = (uint64_t *)((uintptr_t)mbuf2 + + offsetof(struct rte_mempool, + pool_id)); + mbuf3 = (uint64_t *)((uintptr_t)mbuf3 + + offsetof(struct rte_mempool, + pool_id)); + + /* Pick only relevant fields i.e Bit 48:55 of iltype and + * Bit 56:63 of oltype and place it in corresponding + * place in senddesc_w1. + */ + const uint8x16_t shuf_mask0 = { + 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0x6, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xF, 0xE, 0xFF, 0xFF, + }; + + ltypes01 = vqtbl1q_u8(ltypes01, shuf_mask0); + ltypes23 = vqtbl1q_u8(ltypes23, shuf_mask0); + + /* Prepare l4ptr, l3ptr, ol4ptr, ol3ptr from + * l3len, l2len, ol3len, ol2len. + * a [E(32):L3(8):L2(8):OL3(8):OL2(8)] + * a = a + (a << 8) + * a [E:(L3+L2):(L2+OL3):(OL3+OL2):OL2] + * a = a + (a << 16) + * a [E:(L3+L2+OL3+OL2):(L2+OL3+OL2):(OL3+OL2):OL2] + * => E(32):IL4PTR(8):IL3PTR(8):OL4PTR(8):OL3PTR(8) + */ + senddesc01_w1 = vaddq_u8(senddesc01_w1, + vshlq_n_u32(senddesc01_w1, 8)); + senddesc23_w1 = vaddq_u8(senddesc23_w1, + vshlq_n_u32(senddesc23_w1, 8)); + + /* Create second half of 4W cmd for 4 mbufs (sgdesc) */ + cmd01 = vzip1q_u64(sgdesc01_w0, sgdesc01_w1); + cmd11 = vzip2q_u64(sgdesc01_w0, sgdesc01_w1); + cmd21 = vzip1q_u64(sgdesc23_w0, sgdesc23_w1); + cmd31 = vzip2q_u64(sgdesc23_w0, sgdesc23_w1); + + /* Continue preparing l4ptr, l3ptr, ol4ptr, ol3ptr */ + senddesc01_w1 = vaddq_u8( + senddesc01_w1, vshlq_n_u32(senddesc01_w1, 16)); + senddesc23_w1 = vaddq_u8( + senddesc23_w1, vshlq_n_u32(senddesc23_w1, 16)); + + xmask01 = vdupq_n_u64(0); + xmask23 = xmask01; + asm volatile("LD1 {%[a].H}[0],[%[in]]\n\t" + : [a] "+w"(xmask01) + : [in] "r"(mbuf0) + : "memory"); + + asm volatile("LD1 {%[a].H}[4],[%[in]]\n\t" + : [a] "+w"(xmask01) + : [in] "r"(mbuf1) + : "memory"); + + asm volatile("LD1 {%[b].H}[0],[%[in]]\n\t" + : [b] "+w"(xmask23) + : [in] "r"(mbuf2) + : "memory"); + + asm volatile("LD1 {%[b].H}[4],[%[in]]\n\t" + : [b] "+w"(xmask23) + : [in] "r"(mbuf3) + : "memory"); + xmask01 = vshlq_n_u64(xmask01, 20); + xmask23 = vshlq_n_u64(xmask23, 20); + + senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01); + senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23); + /* Move ltypes to senddesc*_w1 */ + senddesc01_w1 = vorrq_u64(senddesc01_w1, ltypes01); + senddesc23_w1 = vorrq_u64(senddesc23_w1, ltypes23); + + /* Create first half of 4W cmd for 4 mbufs (sendhdr) */ + cmd00 = vzip1q_u64(senddesc01_w0, senddesc01_w1); + cmd10 = vzip2q_u64(senddesc01_w0, senddesc01_w1); + cmd20 = vzip1q_u64(senddesc23_w0, senddesc23_w1); + cmd30 = vzip2q_u64(senddesc23_w0, senddesc23_w1); + } else { + /* Just use ld1q to retrieve aura + * when we don't need tx_offload + */ + mbuf0 = (uint64_t *)((uintptr_t)mbuf0 + + offsetof(struct rte_mempool, + pool_id)); + mbuf1 = (uint64_t *)((uintptr_t)mbuf1 + + offsetof(struct rte_mempool, + pool_id)); + mbuf2 = (uint64_t *)((uintptr_t)mbuf2 + + offsetof(struct rte_mempool, + pool_id)); + mbuf3 = (uint64_t *)((uintptr_t)mbuf3 + + offsetof(struct rte_mempool, + pool_id)); + xmask01 = vdupq_n_u64(0); + xmask23 = xmask01; + asm volatile("LD1 {%[a].H}[0],[%[in]]\n\t" + : [a] "+w"(xmask01) + : [in] "r"(mbuf0) + : "memory"); + + asm volatile("LD1 {%[a].H}[4],[%[in]]\n\t" + : [a] "+w"(xmask01) + : [in] "r"(mbuf1) + : "memory"); + + asm volatile("LD1 {%[b].H}[0],[%[in]]\n\t" + : [b] "+w"(xmask23) + : [in] "r"(mbuf2) + : "memory"); + + asm volatile("LD1 {%[b].H}[4],[%[in]]\n\t" + : [b] "+w"(xmask23) + : [in] "r"(mbuf3) + : "memory"); + xmask01 = vshlq_n_u64(xmask01, 20); + xmask23 = vshlq_n_u64(xmask23, 20); + + senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01); + senddesc23_w0 = vorrq_u64(senddesc23_w0, xmask23); + + /* Create 4W cmd for 4 mbufs (sendhdr, sgdesc) */ + cmd00 = vzip1q_u64(senddesc01_w0, senddesc01_w1); + cmd01 = vzip1q_u64(sgdesc01_w0, sgdesc01_w1); + cmd10 = vzip2q_u64(senddesc01_w0, senddesc01_w1); + cmd11 = vzip2q_u64(sgdesc01_w0, sgdesc01_w1); + cmd20 = vzip1q_u64(senddesc23_w0, senddesc23_w1); + cmd21 = vzip1q_u64(sgdesc23_w0, sgdesc23_w1); + cmd30 = vzip2q_u64(senddesc23_w0, senddesc23_w1); + cmd31 = vzip2q_u64(sgdesc23_w0, sgdesc23_w1); + } + + do { + vst1q_u64(lmt_addr, cmd00); + vst1q_u64(lmt_addr + 2, cmd01); + vst1q_u64(lmt_addr + 4, cmd10); + vst1q_u64(lmt_addr + 6, cmd11); + vst1q_u64(lmt_addr + 8, cmd20); + vst1q_u64(lmt_addr + 10, cmd21); + vst1q_u64(lmt_addr + 12, cmd30); + vst1q_u64(lmt_addr + 14, cmd31); + lmt_status = roc_lmt_submit_ldeor(io_addr); + + } while (lmt_status == 0); + tx_pkts = tx_pkts + NIX_DESCS_PER_LOOP; + } + + if (unlikely(pkts_left)) + pkts += nix_xmit_pkts(tx_queue, tx_pkts, pkts_left, cmd, flags); + + return pkts; +} + +#else +static __rte_always_inline uint16_t +nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts, + uint64_t *cmd, const uint16_t flags) +{ + RTE_SET_USED(tx_queue); + RTE_SET_USED(tx_pkts); + RTE_SET_USED(pkts); + RTE_SET_USED(cmd); + RTE_SET_USED(flags); + return 0; +} +#endif + #define T(name, f4, f3, f2, f1, f0, sz, flags) \ static uint16_t __rte_noinline __rte_hot cn9k_nix_xmit_pkts_##name( \ void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t pkts) \ @@ -128,6 +1045,25 @@ NIX_TX_FASTPATH_MODES NIX_TX_FASTPATH_MODES #undef T +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + static uint16_t __rte_noinline __rte_hot \ + cn9k_nix_xmit_pkts_vec_##name(void *tx_queue, \ + struct rte_mbuf **tx_pkts, \ + uint16_t pkts) \ + { \ + uint64_t cmd[sz]; \ + \ + /* VLAN, TSTMP, TSO is not supported by vec */ \ + if ((flags) & NIX_TX_OFFLOAD_VLAN_QINQ_F || \ + (flags) & NIX_TX_OFFLOAD_TSO_F) \ + return 0; \ + return nix_xmit_pkts_vector(tx_queue, tx_pkts, pkts, cmd, \ + (flags)); \ + } + +NIX_TX_FASTPATH_MODES +#undef T + static inline void pick_tx_func(struct rte_eth_dev *eth_dev, const eth_tx_burst_t tx_burst[2][2][2][2][2]) @@ -164,7 +1100,20 @@ cn9k_eth_set_tx_function(struct rte_eth_dev *eth_dev) #undef T }; - pick_tx_func(eth_dev, nix_eth_tx_burst); + const eth_tx_burst_t nix_eth_tx_vec_burst[2][2][2][2][2] = { +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + [f4][f3][f2][f1][f0] = cn9k_nix_xmit_pkts_vec_##name, + + NIX_TX_FASTPATH_MODES +#undef T + }; + + if (dev->scalar_ena || + (dev->tx_offload_flags & + (NIX_TX_OFFLOAD_VLAN_QINQ_F | NIX_TX_OFFLOAD_TSO_F))) + pick_tx_func(eth_dev, nix_eth_tx_burst); + else + pick_tx_func(eth_dev, nix_eth_tx_vec_burst); if (dev->tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) pick_tx_func(eth_dev, nix_eth_tx_burst_mseg); -- 2.8.4