From: Dongdong Liu <liudongdong3@huawei.com>
To: <dev@dpdk.org>, <ferruh.yigit@amd.com>, <thomas@monjalon.net>,
<andrew.rybchenko@oktetlabs.ru>
Cc: <stable@dpdk.org>
Subject: [PATCH 4/5] net/hns3: optimize the rearm mbuf function for SVE Rx
Date: Tue, 11 Jul 2023 18:24:47 +0800 [thread overview]
Message-ID: <20230711102448.11627-5-liudongdong3@huawei.com> (raw)
In-Reply-To: <20230711102448.11627-1-liudongdong3@huawei.com>
From: Huisong Li <lihuisong@huawei.com>
Use hns3_rxq_rearm_mbuf() to replace the hns3_rxq_rearm_mbuf_sve()
to optimize the performance of SVE Rx.
On the rxonly forwarding mode, the performance of a single queue
for 64B packet is improved by ~15%.
Signed-off-by: Huisong Li <lihuisong@huawei.com>
Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
drivers/net/hns3/hns3_rxtx_vec.c | 51 ---------------------------
drivers/net/hns3/hns3_rxtx_vec.h | 51 +++++++++++++++++++++++++++
drivers/net/hns3/hns3_rxtx_vec_sve.c | 52 ++--------------------------
3 files changed, 53 insertions(+), 101 deletions(-)
diff --git a/drivers/net/hns3/hns3_rxtx_vec.c b/drivers/net/hns3/hns3_rxtx_vec.c
index cd9264d91b..9708ec614e 100644
--- a/drivers/net/hns3/hns3_rxtx_vec.c
+++ b/drivers/net/hns3/hns3_rxtx_vec.c
@@ -55,57 +55,6 @@ hns3_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
return nb_tx;
}
-static inline void
-hns3_rxq_rearm_mbuf(struct hns3_rx_queue *rxq)
-{
-#define REARM_LOOP_STEP_NUM 4
- struct hns3_entry *rxep = &rxq->sw_ring[rxq->rx_rearm_start];
- struct hns3_desc *rxdp = rxq->rx_ring + rxq->rx_rearm_start;
- uint64_t dma_addr;
- int i;
-
- if (unlikely(rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
- HNS3_DEFAULT_RXQ_REARM_THRESH) < 0)) {
- rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
- return;
- }
-
- for (i = 0; i < HNS3_DEFAULT_RXQ_REARM_THRESH; i += REARM_LOOP_STEP_NUM,
- rxep += REARM_LOOP_STEP_NUM, rxdp += REARM_LOOP_STEP_NUM) {
- if (likely(i <
- HNS3_DEFAULT_RXQ_REARM_THRESH - REARM_LOOP_STEP_NUM)) {
- rte_prefetch_non_temporal(rxep[4].mbuf);
- rte_prefetch_non_temporal(rxep[5].mbuf);
- rte_prefetch_non_temporal(rxep[6].mbuf);
- rte_prefetch_non_temporal(rxep[7].mbuf);
- }
-
- dma_addr = rte_mbuf_data_iova_default(rxep[0].mbuf);
- rxdp[0].addr = rte_cpu_to_le_64(dma_addr);
- rxdp[0].rx.bd_base_info = 0;
-
- dma_addr = rte_mbuf_data_iova_default(rxep[1].mbuf);
- rxdp[1].addr = rte_cpu_to_le_64(dma_addr);
- rxdp[1].rx.bd_base_info = 0;
-
- dma_addr = rte_mbuf_data_iova_default(rxep[2].mbuf);
- rxdp[2].addr = rte_cpu_to_le_64(dma_addr);
- rxdp[2].rx.bd_base_info = 0;
-
- dma_addr = rte_mbuf_data_iova_default(rxep[3].mbuf);
- rxdp[3].addr = rte_cpu_to_le_64(dma_addr);
- rxdp[3].rx.bd_base_info = 0;
- }
-
- rxq->rx_rearm_start += HNS3_DEFAULT_RXQ_REARM_THRESH;
- if (rxq->rx_rearm_start >= rxq->nb_rx_desc)
- rxq->rx_rearm_start = 0;
-
- rxq->rx_rearm_nb -= HNS3_DEFAULT_RXQ_REARM_THRESH;
-
- hns3_write_reg_opt(rxq->io_head_reg, HNS3_DEFAULT_RXQ_REARM_THRESH);
-}
-
uint16_t
hns3_recv_pkts_vec(void *__restrict rx_queue,
struct rte_mbuf **__restrict rx_pkts,
diff --git a/drivers/net/hns3/hns3_rxtx_vec.h b/drivers/net/hns3/hns3_rxtx_vec.h
index 2c8a91921e..a9a6774294 100644
--- a/drivers/net/hns3/hns3_rxtx_vec.h
+++ b/drivers/net/hns3/hns3_rxtx_vec.h
@@ -94,4 +94,55 @@ hns3_rx_reassemble_pkts(struct rte_mbuf **rx_pkts,
return count;
}
+
+static inline void
+hns3_rxq_rearm_mbuf(struct hns3_rx_queue *rxq)
+{
+#define REARM_LOOP_STEP_NUM 4
+ struct hns3_entry *rxep = &rxq->sw_ring[rxq->rx_rearm_start];
+ struct hns3_desc *rxdp = rxq->rx_ring + rxq->rx_rearm_start;
+ uint64_t dma_addr;
+ int i;
+
+ if (unlikely(rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
+ HNS3_DEFAULT_RXQ_REARM_THRESH) < 0)) {
+ rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
+ return;
+ }
+
+ for (i = 0; i < HNS3_DEFAULT_RXQ_REARM_THRESH; i += REARM_LOOP_STEP_NUM,
+ rxep += REARM_LOOP_STEP_NUM, rxdp += REARM_LOOP_STEP_NUM) {
+ if (likely(i <
+ HNS3_DEFAULT_RXQ_REARM_THRESH - REARM_LOOP_STEP_NUM)) {
+ rte_prefetch_non_temporal(rxep[4].mbuf);
+ rte_prefetch_non_temporal(rxep[5].mbuf);
+ rte_prefetch_non_temporal(rxep[6].mbuf);
+ rte_prefetch_non_temporal(rxep[7].mbuf);
+ }
+
+ dma_addr = rte_mbuf_data_iova_default(rxep[0].mbuf);
+ rxdp[0].addr = rte_cpu_to_le_64(dma_addr);
+ rxdp[0].rx.bd_base_info = 0;
+
+ dma_addr = rte_mbuf_data_iova_default(rxep[1].mbuf);
+ rxdp[1].addr = rte_cpu_to_le_64(dma_addr);
+ rxdp[1].rx.bd_base_info = 0;
+
+ dma_addr = rte_mbuf_data_iova_default(rxep[2].mbuf);
+ rxdp[2].addr = rte_cpu_to_le_64(dma_addr);
+ rxdp[2].rx.bd_base_info = 0;
+
+ dma_addr = rte_mbuf_data_iova_default(rxep[3].mbuf);
+ rxdp[3].addr = rte_cpu_to_le_64(dma_addr);
+ rxdp[3].rx.bd_base_info = 0;
+ }
+
+ rxq->rx_rearm_start += HNS3_DEFAULT_RXQ_REARM_THRESH;
+ if (rxq->rx_rearm_start >= rxq->nb_rx_desc)
+ rxq->rx_rearm_start = 0;
+
+ rxq->rx_rearm_nb -= HNS3_DEFAULT_RXQ_REARM_THRESH;
+
+ hns3_write_reg_opt(rxq->io_head_reg, HNS3_DEFAULT_RXQ_REARM_THRESH);
+}
#endif /* HNS3_RXTX_VEC_H */
diff --git a/drivers/net/hns3/hns3_rxtx_vec_sve.c b/drivers/net/hns3/hns3_rxtx_vec_sve.c
index 5011544e07..54aef7db8d 100644
--- a/drivers/net/hns3/hns3_rxtx_vec_sve.c
+++ b/drivers/net/hns3/hns3_rxtx_vec_sve.c
@@ -237,54 +237,6 @@ hns3_recv_burst_vec_sve(struct hns3_rx_queue *__restrict rxq,
return nb_rx;
}
-static inline void
-hns3_rxq_rearm_mbuf_sve(struct hns3_rx_queue *rxq)
-{
-#define REARM_LOOP_STEP_NUM 4
- struct hns3_entry *rxep = &rxq->sw_ring[rxq->rx_rearm_start];
- struct hns3_desc *rxdp = rxq->rx_ring + rxq->rx_rearm_start;
- struct hns3_entry *rxep_tmp = rxep;
- int i;
-
- if (unlikely(rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep,
- HNS3_DEFAULT_RXQ_REARM_THRESH) < 0)) {
- rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
- return;
- }
-
- for (i = 0; i < HNS3_DEFAULT_RXQ_REARM_THRESH; i += REARM_LOOP_STEP_NUM,
- rxep_tmp += REARM_LOOP_STEP_NUM) {
- svuint64_t prf = svld1_u64(PG64_256BIT, (uint64_t *)rxep_tmp);
- svprfd_gather_u64base(PG64_256BIT, prf, SV_PLDL1STRM);
- }
-
- for (i = 0; i < HNS3_DEFAULT_RXQ_REARM_THRESH; i += REARM_LOOP_STEP_NUM,
- rxep += REARM_LOOP_STEP_NUM, rxdp += REARM_LOOP_STEP_NUM) {
- uint64_t iova[REARM_LOOP_STEP_NUM];
- iova[0] = rte_mbuf_iova_get(rxep[0].mbuf);
- iova[1] = rte_mbuf_iova_get(rxep[1].mbuf);
- iova[2] = rte_mbuf_iova_get(rxep[2].mbuf);
- iova[3] = rte_mbuf_iova_get(rxep[3].mbuf);
- svuint64_t siova = svld1_u64(PG64_256BIT, iova);
- siova = svadd_n_u64_z(PG64_256BIT, siova, RTE_PKTMBUF_HEADROOM);
- svuint64_t ol_base = svdup_n_u64(0);
- svst1_scatter_u64offset_u64(PG64_256BIT,
- (uint64_t *)&rxdp[0].addr,
- svindex_u64(BD_FIELD_ADDR_OFFSET, BD_SIZE), siova);
- svst1_scatter_u64offset_u64(PG64_256BIT,
- (uint64_t *)&rxdp[0].addr,
- svindex_u64(BD_FIELD_OL_OFFSET, BD_SIZE), ol_base);
- }
-
- rxq->rx_rearm_start += HNS3_DEFAULT_RXQ_REARM_THRESH;
- if (rxq->rx_rearm_start >= rxq->nb_rx_desc)
- rxq->rx_rearm_start = 0;
-
- rxq->rx_rearm_nb -= HNS3_DEFAULT_RXQ_REARM_THRESH;
-
- hns3_write_reg_opt(rxq->io_head_reg, HNS3_DEFAULT_RXQ_REARM_THRESH);
-}
-
uint16_t
hns3_recv_pkts_vec_sve(void *__restrict rx_queue,
struct rte_mbuf **__restrict rx_pkts,
@@ -300,7 +252,7 @@ hns3_recv_pkts_vec_sve(void *__restrict rx_queue,
nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, HNS3_SVE_DEFAULT_DESCS_PER_LOOP);
if (rxq->rx_rearm_nb > HNS3_DEFAULT_RXQ_REARM_THRESH)
- hns3_rxq_rearm_mbuf_sve(rxq);
+ hns3_rxq_rearm_mbuf(rxq);
if (unlikely(!(rxdp->rx.bd_base_info &
rte_cpu_to_le_32(1u << HNS3_RXD_VLD_B))))
@@ -331,7 +283,7 @@ hns3_recv_pkts_vec_sve(void *__restrict rx_queue,
break;
if (rxq->rx_rearm_nb > HNS3_DEFAULT_RXQ_REARM_THRESH)
- hns3_rxq_rearm_mbuf_sve(rxq);
+ hns3_rxq_rearm_mbuf(rxq);
}
return nb_rx;
--
2.22.0
next prev parent reply other threads:[~2023-07-11 10:28 UTC|newest]
Thread overview: 14+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-07-11 10:24 [PATCH 0/5] net/hns3: some performance optimizations Dongdong Liu
2023-07-11 10:24 ` [PATCH 1/5] net/hns3: fix incorrect index to look up table in NEON Rx Dongdong Liu
2023-07-11 12:58 ` Ferruh Yigit
2023-07-11 10:24 ` [PATCH 2/5] net/hns3: fix the order of NEON Rx code Dongdong Liu
2023-07-11 10:24 ` [PATCH 3/5] net/hns3: optimize free mbuf code for SVE Tx Dongdong Liu
2023-09-25 14:21 ` Ferruh Yigit
2023-09-26 4:03 ` lihuisong (C)
2023-07-11 10:24 ` Dongdong Liu [this message]
2023-07-11 10:24 ` [PATCH 5/5] net/hns3: optimize SVE Rx performance Dongdong Liu
2023-07-11 10:48 ` [PATCH 0/5] net/hns3: some performance optimizations Ferruh Yigit
2023-07-11 11:27 ` Dongdong Liu
2023-07-11 12:26 ` Ferruh Yigit
2023-09-25 14:26 ` Ferruh Yigit
2023-09-25 2:33 ` Jie Hai
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230711102448.11627-5-liudongdong3@huawei.com \
--to=liudongdong3@huawei.com \
--cc=andrew.rybchenko@oktetlabs.ru \
--cc=dev@dpdk.org \
--cc=ferruh.yigit@amd.com \
--cc=stable@dpdk.org \
--cc=thomas@monjalon.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).