From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 32A03A04C7; Fri, 18 Sep 2020 05:54:03 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 1E4321D739; Fri, 18 Sep 2020 05:53:46 +0200 (CEST) Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id A79D81D725 for ; Fri, 18 Sep 2020 05:53:41 +0200 (CEST) IronPort-SDR: qc/trwVOGS8cPJfZ445kvV8fgaEo89fdmhBKsKe2qDH9OlXQGG4bfXkOliFrS74SzUd3w08qoy 6eW7TBmnPgKQ== X-IronPort-AV: E=McAfee;i="6000,8403,9747"; a="159155196" X-IronPort-AV: E=Sophos;i="5.77,273,1596524400"; d="scan'208";a="159155196" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by fmsmga103.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 17 Sep 2020 20:53:41 -0700 IronPort-SDR: PSjKCYq1Z8j27LbK/08zSMpTpBQf2ox38wqYqaUj2oahlgM4M9rYrHmG0EQF3VFGpXU7L/Xsm1 RtLGsuAl7qeg== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.77,273,1596524400"; d="scan'208";a="307728648" Received: from dpdk-lrong-srv-04.sh.intel.com ([10.67.119.221]) by orsmga006.jf.intel.com with ESMTP; 17 Sep 2020 20:53:39 -0700 From: Leyi Rong To: qi.z.zhang@intel.com, wenzhuo.lu@intel.com, burce.richardson@intel.com Cc: dev@dpdk.org, Leyi Rong , Bruce Richardson Date: Fri, 18 Sep 2020 11:35:28 +0800 Message-Id: <20200918033528.110297-4-leyi.rong@intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20200918033528.110297-1-leyi.rong@intel.com> References: <20200910065504.104217-1-leyi.rong@intel.com> <20200918033528.110297-1-leyi.rong@intel.com> Subject: [dpdk-dev] [PATCH v2 3/3] net/ice: optimize Tx path on AVX512 vPMD X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Optimize Tx path by using AVX512 instructions and vectorize the tx free bufs process. Signed-off-by: Leyi Rong Signed-off-by: Bruce Richardson --- drivers/net/ice/ice_rxtx.h | 4 + drivers/net/ice/ice_rxtx_vec_avx512.c | 155 +++++++++++++++++++++----- drivers/net/ice/ice_rxtx_vec_common.h | 36 ++++-- 3 files changed, 162 insertions(+), 33 deletions(-) diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h index c13e54b251..6eb3447d26 100644 --- a/drivers/net/ice/ice_rxtx.h +++ b/drivers/net/ice/ice_rxtx.h @@ -91,6 +91,10 @@ struct ice_tx_entry { uint16_t last_id; }; +struct ice_vec_tx_entry { + struct rte_mbuf *mbuf; +}; + struct ice_tx_queue { uint16_t nb_tx_desc; /* number of TX descriptors */ rte_iova_t tx_ring_dma; /* TX ring DMA address */ diff --git a/drivers/net/ice/ice_rxtx_vec_avx512.c b/drivers/net/ice/ice_rxtx_vec_avx512.c index a2a5d9987a..e5e7cc1482 100644 --- a/drivers/net/ice/ice_rxtx_vec_avx512.c +++ b/drivers/net/ice/ice_rxtx_vec_avx512.c @@ -756,6 +756,108 @@ ice_recv_scattered_pkts_vec_avx512(void *rx_queue, struct rte_mbuf **rx_pkts, rx_pkts + retval, nb_pkts); } +static __rte_always_inline int +ice_tx_free_bufs_avx512(struct ice_tx_queue *txq) +{ + struct ice_vec_tx_entry *txep; + uint32_t n; + uint32_t i; + int nb_free = 0; + struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ]; + + /* check DD bits on threshold descriptor */ + if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & + rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) != + rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE)) + return 0; + + n = txq->tx_rs_thresh; + + /* first buffer to free from S/W ring is at index + * tx_next_dd - (tx_rs_thresh - 1) + */ + txep = (void *)txq->sw_ring; + txep += txq->tx_next_dd - (n - 1); + + if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE && (n & 31) == 0) { + struct rte_mempool *mp = txep[0].mbuf->pool; + struct rte_mempool_cache *cache = rte_mempool_default_cache(mp, + rte_lcore_id()); + void **cache_objs = &cache->objs[cache->len]; + + if (n > RTE_MEMPOOL_CACHE_MAX_SIZE) { + rte_mempool_ops_enqueue_bulk(mp, (void *)txep, n); + goto done; + } + + /* The cache follows the following algorithm + * 1. Add the objects to the cache + * 2. Anything greater than the cache min value (if it + * crosses the cache flush threshold) is flushed to the ring. + */ + /* Add elements back into the cache */ + uint32_t copied = 0; + /* n is multiple of 32 */ + while (copied < n) { + const __m512i a = _mm512_loadu_si512(&txep[copied]); + const __m512i b = _mm512_loadu_si512(&txep[copied + 8]); + const __m512i c = _mm512_loadu_si512(&txep[copied + 16]); + const __m512i d = _mm512_loadu_si512(&txep[copied + 24]); + + _mm512_storeu_si512(&cache_objs[copied], a); + _mm512_storeu_si512(&cache_objs[copied + 8], b); + _mm512_storeu_si512(&cache_objs[copied + 16], c); + _mm512_storeu_si512(&cache_objs[copied + 24], d); + copied += 32; + } + cache->len += n; + + if (cache->len >= cache->flushthresh) { + rte_mempool_ops_enqueue_bulk + (mp, &cache->objs[cache->size], + cache->len - cache->size); + cache->len = cache->size; + } + goto done; + } + + m = rte_pktmbuf_prefree_seg(txep[0].mbuf); + if (likely(m)) { + free[0] = m; + nb_free = 1; + for (i = 1; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (likely(m)) { + if (likely(m->pool == free[0]->pool)) { + free[nb_free++] = m; + } else { + rte_mempool_put_bulk(free[0]->pool, + (void *)free, + nb_free); + free[0] = m; + nb_free = 1; + } + } + } + rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); + } else { + for (i = 1; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (m) + rte_mempool_put(m->pool, m); + } + } + +done: + /* buffers were freed, update counters */ + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); + if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); + + return txq->tx_rs_thresh; +} + static inline void ice_vtx1(volatile struct ice_tx_desc *txdp, struct rte_mbuf *pkt, uint64_t flags) @@ -766,7 +868,7 @@ ice_vtx1(volatile struct ice_tx_desc *txdp, ((uint64_t)pkt->data_len << ICE_TXD_QW1_TX_BUF_SZ_S)); __m128i descriptor = _mm_set_epi64x(high_qw, - pkt->buf_physaddr + pkt->data_off); + pkt->buf_iova + pkt->data_off); _mm_store_si128((__m128i *)txdp, descriptor); } @@ -777,13 +879,6 @@ ice_vtx(volatile struct ice_tx_desc *txdp, const uint64_t hi_qw_tmpl = (ICE_TX_DESC_DTYPE_DATA | ((uint64_t)flags << ICE_TXD_QW1_CMD_S)); - /* if unaligned on 32-bit boundary, do one to align */ - if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) { - ice_vtx1(txdp, *pkt, flags); - nb_pkts--, txdp++, pkt++; - } - - /* do two at a time while possible, in bursts */ for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) { uint64_t hi_qw3 = hi_qw_tmpl | @@ -802,20 +897,17 @@ ice_vtx(volatile struct ice_tx_desc *txdp, ((uint64_t)pkt[0]->data_len << ICE_TXD_QW1_TX_BUF_SZ_S); - __m256i desc2_3 = - _mm256_set_epi64x + __m512i desc0_3 = + _mm512_set_epi64 (hi_qw3, - pkt[3]->buf_physaddr + pkt[3]->data_off, + pkt[3]->buf_iova + pkt[3]->data_off, hi_qw2, - pkt[2]->buf_physaddr + pkt[2]->data_off); - __m256i desc0_1 = - _mm256_set_epi64x - (hi_qw1, - pkt[1]->buf_physaddr + pkt[1]->data_off, + pkt[2]->buf_iova + pkt[2]->data_off, + hi_qw1, + pkt[1]->buf_iova + pkt[1]->data_off, hi_qw0, - pkt[0]->buf_physaddr + pkt[0]->data_off); - _mm256_store_si256((void *)(txdp + 2), desc2_3); - _mm256_store_si256((void *)txdp, desc0_1); + pkt[0]->buf_iova + pkt[0]->data_off); + _mm512_storeu_si512((void *)txdp, desc0_3); } /* do any last ones */ @@ -825,13 +917,23 @@ ice_vtx(volatile struct ice_tx_desc *txdp, } } +static __rte_always_inline void +ice_tx_backlog_entry_avx512(struct ice_vec_tx_entry *txep, + struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + int i; + + for (i = 0; i < (int)nb_pkts; ++i) + txep[i].mbuf = tx_pkts[i]; +} + static inline uint16_t ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; volatile struct ice_tx_desc *txdp; - struct ice_tx_entry *txep; + struct ice_vec_tx_entry *txep; uint16_t n, nb_commit, tx_id; uint64_t flags = ICE_TD_CMD; uint64_t rs = ICE_TX_DESC_CMD_RS | ICE_TD_CMD; @@ -840,7 +942,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); if (txq->nb_tx_free < txq->tx_free_thresh) - ice_tx_free_bufs(txq); + ice_tx_free_bufs_avx512(txq); nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); if (unlikely(nb_pkts == 0)) @@ -848,13 +950,14 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txep = (void *)txq->sw_ring; + txep += tx_id; txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); n = (uint16_t)(txq->nb_tx_desc - tx_id); if (nb_commit >= n) { - ice_tx_backlog_entry(txep, tx_pkts, n); + ice_tx_backlog_entry_avx512(txep, tx_pkts, n); ice_vtx(txdp, tx_pkts, n - 1, flags); tx_pkts += (n - 1); @@ -868,11 +971,11 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); /* avoid reach the end of ring */ - txdp = &txq->tx_ring[tx_id]; - txep = &txq->sw_ring[tx_id]; + txdp = txq->tx_ring; + txep = (void *)txq->sw_ring; } - ice_tx_backlog_entry(txep, tx_pkts, nb_commit); + ice_tx_backlog_entry_avx512(txep, tx_pkts, nb_commit); ice_vtx(txdp, tx_pkts, nb_commit, flags); diff --git a/drivers/net/ice/ice_rxtx_vec_common.h b/drivers/net/ice/ice_rxtx_vec_common.h index e2019c8d6a..ae2ac29f2a 100644 --- a/drivers/net/ice/ice_rxtx_vec_common.h +++ b/drivers/net/ice/ice_rxtx_vec_common.h @@ -189,16 +189,38 @@ _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) * so need to free remains more carefully. */ i = txq->tx_next_dd - txq->tx_rs_thresh + 1; - if (txq->tx_tail < i) { - for (; i < txq->nb_tx_desc; i++) { + +#ifdef CC_AVX512_SUPPORT + struct rte_eth_dev *dev = txq->vsi->adapter->eth_dev; + + if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512) { + struct ice_vec_tx_entry *swr = (void *)txq->sw_ring; + + if (txq->tx_tail < i) { + for (; i < txq->nb_tx_desc; i++) { + rte_pktmbuf_free_seg(swr[i].mbuf); + swr[i].mbuf = NULL; + } + i = 0; + } + for (; i < txq->tx_tail; i++) { + rte_pktmbuf_free_seg(swr[i].mbuf); + swr[i].mbuf = NULL; + } + } else +#endif + { + if (txq->tx_tail < i) { + for (; i < txq->nb_tx_desc; i++) { + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); + txq->sw_ring[i].mbuf = NULL; + } + i = 0; + } + for (; i < txq->tx_tail; i++) { rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); txq->sw_ring[i].mbuf = NULL; } - i = 0; - } - for (; i < txq->tx_tail; i++) { - rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); - txq->sw_ring[i].mbuf = NULL; } } -- 2.17.1