From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id E795D430EC; Thu, 24 Aug 2023 09:37:04 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id C0BD843036; Thu, 24 Aug 2023 09:36:52 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id CDC4740EE1 for ; Thu, 24 Aug 2023 09:36:50 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 8ACB311FB; Thu, 24 Aug 2023 00:37:30 -0700 (PDT) Received: from net-x86-dell-8268.shanghai.arm.com (net-x86-dell-8268.shanghai.arm.com [10.169.210.116]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id CC4803F762; Thu, 24 Aug 2023 00:36:47 -0700 (PDT) From: Feifei Wang To: Qiming Yang , Wenjun Wu Cc: dev@dpdk.org, nd@arm.com, Feifei Wang , Honnappa Nagarahalli , Ruifeng Wang Subject: [PATCH v12 3/4] net/ixgbe: implement mbufs recycle mode Date: Thu, 24 Aug 2023 15:36:28 +0800 Message-Id: <20230824073629.2577864-4-feifei.wang2@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230824073629.2577864-1-feifei.wang2@arm.com> References: <20220420081650.2043183-1-feifei.wang2@arm.com> <20230824073629.2577864-1-feifei.wang2@arm.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Define specific function implementation for ixgbe driver. Currently, recycle buffer mode can support 128bit vector path. And can be enabled both in fast free and no fast free mode. Suggested-by: Honnappa Nagarahalli Signed-off-by: Feifei Wang Reviewed-by: Ruifeng Wang Reviewed-by: Honnappa Nagarahalli --- drivers/net/ixgbe/ixgbe_ethdev.c | 1 + drivers/net/ixgbe/ixgbe_ethdev.h | 3 + .../ixgbe/ixgbe_recycle_mbufs_vec_common.c | 143 ++++++++++++++++++ drivers/net/ixgbe/ixgbe_rxtx.c | 37 ++++- drivers/net/ixgbe/ixgbe_rxtx.h | 4 + drivers/net/ixgbe/meson.build | 2 + 6 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c diff --git a/drivers/net/ixgbe/ixgbe_ethdev.c b/drivers/net/ixgbe/ixgbe_ethdev.c index 14a7d571e0..ea4c9dd561 100644 --- a/drivers/net/ixgbe/ixgbe_ethdev.c +++ b/drivers/net/ixgbe/ixgbe_ethdev.c @@ -543,6 +543,7 @@ static const struct eth_dev_ops ixgbe_eth_dev_ops = { .set_mc_addr_list = ixgbe_dev_set_mc_addr_list, .rxq_info_get = ixgbe_rxq_info_get, .txq_info_get = ixgbe_txq_info_get, + .recycle_rxq_info_get = ixgbe_recycle_rxq_info_get, .timesync_enable = ixgbe_timesync_enable, .timesync_disable = ixgbe_timesync_disable, .timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp, diff --git a/drivers/net/ixgbe/ixgbe_ethdev.h b/drivers/net/ixgbe/ixgbe_ethdev.h index 1291e9099c..22fc3be3d8 100644 --- a/drivers/net/ixgbe/ixgbe_ethdev.h +++ b/drivers/net/ixgbe/ixgbe_ethdev.h @@ -626,6 +626,9 @@ void ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, void ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_txq_info *qinfo); +void ixgbe_recycle_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_recycle_rxq_info *recycle_rxq_info); + int ixgbevf_dev_rx_init(struct rte_eth_dev *dev); void ixgbevf_dev_tx_init(struct rte_eth_dev *dev); diff --git a/drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c b/drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c new file mode 100644 index 0000000000..9a8cc86954 --- /dev/null +++ b/drivers/net/ixgbe/ixgbe_recycle_mbufs_vec_common.c @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Arm Limited. + */ + +#include +#include + +#include "ixgbe_ethdev.h" +#include "ixgbe_rxtx.h" + +#pragma GCC diagnostic ignored "-Wcast-qual" + +void +ixgbe_recycle_rx_descriptors_refill_vec(void *rx_queue, uint16_t nb_mbufs) +{ + struct ixgbe_rx_queue *rxq = rx_queue; + struct ixgbe_rx_entry *rxep; + volatile union ixgbe_adv_rx_desc *rxdp; + uint16_t rx_id; + uint64_t paddr; + uint64_t dma_addr; + uint16_t i; + + rxdp = rxq->rx_ring + rxq->rxrearm_start; + rxep = &rxq->sw_ring[rxq->rxrearm_start]; + + for (i = 0; i < nb_mbufs; i++) { + /* Initialize rxdp descs. */ + paddr = (rxep[i].mbuf)->buf_iova + RTE_PKTMBUF_HEADROOM; + dma_addr = rte_cpu_to_le_64(paddr); + /* Flush descriptors with pa dma_addr */ + rxdp[i].read.hdr_addr = 0; + rxdp[i].read.pkt_addr = dma_addr; + } + + /* Update the descriptor initializer index */ + rxq->rxrearm_start += nb_mbufs; + if (rxq->rxrearm_start >= rxq->nb_rx_desc) + rxq->rxrearm_start = 0; + + rxq->rxrearm_nb -= nb_mbufs; + + rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? + (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); + + /* Update the tail pointer on the NIC */ + IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); +} + +uint16_t +ixgbe_recycle_tx_mbufs_reuse_vec(void *tx_queue, + struct rte_eth_recycle_rxq_info *recycle_rxq_info) +{ + struct ixgbe_tx_queue *txq = tx_queue; + struct ixgbe_tx_entry *txep; + struct rte_mbuf **rxep; + int i, n; + uint32_t status; + uint16_t nb_recycle_mbufs; + uint16_t avail = 0; + uint16_t mbuf_ring_size = recycle_rxq_info->mbuf_ring_size; + uint16_t mask = recycle_rxq_info->mbuf_ring_size - 1; + uint16_t refill_requirement = recycle_rxq_info->refill_requirement; + uint16_t refill_head = *recycle_rxq_info->refill_head; + uint16_t receive_tail = *recycle_rxq_info->receive_tail; + + /* Get available recycling Rx buffers. */ + avail = (mbuf_ring_size - (refill_head - receive_tail)) & mask; + + /* Check Tx free thresh and Rx available space. */ + if (txq->nb_tx_free > txq->tx_free_thresh || avail <= txq->tx_rs_thresh) + return 0; + + /* check DD bits on threshold descriptor */ + status = txq->tx_ring[txq->tx_next_dd].wb.status; + if (!(status & IXGBE_ADVTXD_STAT_DD)) + return 0; + + n = txq->tx_rs_thresh; + nb_recycle_mbufs = n; + + /* Mbufs recycle can only support no ring buffer wrapping around. + * Two case for this: + * + * case 1: The refill head of Rx buffer ring needs to be aligned with + * buffer ring size. In this case, the number of Tx freeing buffers + * should be equal to refill_requirement. + * + * case 2: The refill head of Rx ring buffer does not need to be aligned + * with buffer ring size. In this case, the update of refill head can not + * exceed the Rx buffer ring size. + */ + if (refill_requirement != n || + (!refill_requirement && (refill_head + n > mbuf_ring_size))) + return 0; + + /* First buffer to free from S/W ring is at index + * tx_next_dd - (tx_rs_thresh-1). + */ + txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; + rxep = recycle_rxq_info->mbuf_ring; + rxep += refill_head; + + if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { + /* Avoid txq contains buffers from unexpected mempool. */ + if (unlikely(recycle_rxq_info->mp + != txep[0].mbuf->pool)) + return 0; + + /* Directly put mbufs from Tx to Rx. */ + for (i = 0; i < n; i++) + rxep[i] = txep[i].mbuf; + } else { + for (i = 0; i < n; i++) { + rxep[i] = rte_pktmbuf_prefree_seg(txep[i].mbuf); + + /* If Tx buffers are not the last reference or from + * unexpected mempool, previous copied buffers are + * considered as invalid. + */ + if (unlikely((rxep[i] == NULL && refill_requirement) || + recycle_rxq_info->mp != txep[i].mbuf->pool)) + nb_recycle_mbufs = 0; + } + /* If Tx buffers are not the last reference or + * from unexpected mempool, all recycled buffers + * are put into mempool. + */ + if (nb_recycle_mbufs == 0) + for (i = 0; i < n; i++) { + if (rxep[i] != NULL) + rte_mempool_put(rxep[i]->pool, rxep[i]); + } + } + + /* Update counters for Tx. */ + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); + if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); + + return nb_recycle_mbufs; +} diff --git a/drivers/net/ixgbe/ixgbe_rxtx.c b/drivers/net/ixgbe/ixgbe_rxtx.c index 954ef241a0..90b0a7004f 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx.c +++ b/drivers/net/ixgbe/ixgbe_rxtx.c @@ -2552,6 +2552,9 @@ ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq) (rte_eal_process_type() != RTE_PROC_PRIMARY || ixgbe_txq_vec_setup(txq) == 0)) { PMD_INIT_LOG(DEBUG, "Vector tx enabled."); +#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) + dev->recycle_tx_mbufs_reuse = ixgbe_recycle_tx_mbufs_reuse_vec; +#endif dev->tx_pkt_burst = ixgbe_xmit_pkts_vec; } else dev->tx_pkt_burst = ixgbe_xmit_pkts_simple; @@ -4890,7 +4893,10 @@ ixgbe_set_rx_function(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx " "callback (port=%d).", dev->data->port_id); - +#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) + dev->recycle_rx_descriptors_refill = + ixgbe_recycle_rx_descriptors_refill_vec; +#endif dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec; } else if (adapter->rx_bulk_alloc_allowed) { PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " @@ -4919,7 +4925,9 @@ ixgbe_set_rx_function(struct rte_eth_dev *dev) "burst size no less than %d (port=%d).", RTE_IXGBE_DESCS_PER_LOOP, dev->data->port_id); - +#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) + dev->recycle_rx_descriptors_refill = ixgbe_recycle_rx_descriptors_refill_vec; +#endif dev->rx_pkt_burst = ixgbe_recv_pkts_vec; } else if (adapter->rx_bulk_alloc_allowed) { PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " @@ -5691,6 +5699,31 @@ ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.tx_deferred_start = txq->tx_deferred_start; } +void +ixgbe_recycle_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_recycle_rxq_info *recycle_rxq_info) +{ + struct ixgbe_rx_queue *rxq; + struct ixgbe_adapter *adapter = dev->data->dev_private; + + rxq = dev->data->rx_queues[queue_id]; + + recycle_rxq_info->mbuf_ring = (void *)rxq->sw_ring; + recycle_rxq_info->mp = rxq->mb_pool; + recycle_rxq_info->mbuf_ring_size = rxq->nb_rx_desc; + recycle_rxq_info->receive_tail = &rxq->rx_tail; + + if (adapter->rx_vec_allowed) { +#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) + recycle_rxq_info->refill_requirement = RTE_IXGBE_RXQ_REARM_THRESH; + recycle_rxq_info->refill_head = &rxq->rxrearm_start; +#endif + } else { + recycle_rxq_info->refill_requirement = rxq->rx_free_thresh; + recycle_rxq_info->refill_head = &rxq->rx_free_trigger; + } +} + /* * [VF] Initializes Receive Unit. */ diff --git a/drivers/net/ixgbe/ixgbe_rxtx.h b/drivers/net/ixgbe/ixgbe_rxtx.h index 668a5b9814..ee89c89929 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx.h +++ b/drivers/net/ixgbe/ixgbe_rxtx.h @@ -295,6 +295,10 @@ int ixgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt); extern const uint32_t ptype_table[IXGBE_PACKET_TYPE_MAX]; extern const uint32_t ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX]; +uint16_t ixgbe_recycle_tx_mbufs_reuse_vec(void *tx_queue, + struct rte_eth_recycle_rxq_info *recycle_rxq_info); +void ixgbe_recycle_rx_descriptors_refill_vec(void *rx_queue, uint16_t nb_mbufs); + uint16_t ixgbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); int ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq); diff --git a/drivers/net/ixgbe/meson.build b/drivers/net/ixgbe/meson.build index a18908ef7c..0ae12dd5ff 100644 --- a/drivers/net/ixgbe/meson.build +++ b/drivers/net/ixgbe/meson.build @@ -26,11 +26,13 @@ deps += ['hash', 'security'] if arch_subdir == 'x86' sources += files('ixgbe_rxtx_vec_sse.c') + sources += files('ixgbe_recycle_mbufs_vec_common.c') if is_windows and cc.get_id() != 'clang' cflags += ['-fno-asynchronous-unwind-tables'] endif elif arch_subdir == 'arm' sources += files('ixgbe_rxtx_vec_neon.c') + sources += files('ixgbe_recycle_mbufs_vec_common.c') endif includes += include_directories('base') -- 2.25.1