From: Leyi Rong <leyi.rong@intel.com>
To: qi.z.zhang@intel.com, wenzhuo.lu@intel.com, burce.richardson@intel.com
Cc: dev@dpdk.org, Leyi Rong <leyi.rong@intel.com>,
Bruce Richardson <bruce.richardson@intel.com>
Subject: [dpdk-dev] [PATCH v2 3/3] net/ice: optimize Tx path on AVX512 vPMD
Date: Fri, 18 Sep 2020 11:35:28 +0800 [thread overview]
Message-ID: <20200918033528.110297-4-leyi.rong@intel.com> (raw)
In-Reply-To: <20200918033528.110297-1-leyi.rong@intel.com>
Optimize Tx path by using AVX512 instructions and vectorize the
tx free bufs process.
Signed-off-by: Leyi Rong <leyi.rong@intel.com>
Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
drivers/net/ice/ice_rxtx.h | 4 +
drivers/net/ice/ice_rxtx_vec_avx512.c | 155 +++++++++++++++++++++-----
drivers/net/ice/ice_rxtx_vec_common.h | 36 ++++--
3 files changed, 162 insertions(+), 33 deletions(-)
diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h
index c13e54b251..6eb3447d26 100644
--- a/drivers/net/ice/ice_rxtx.h
+++ b/drivers/net/ice/ice_rxtx.h
@@ -91,6 +91,10 @@ struct ice_tx_entry {
uint16_t last_id;
};
+struct ice_vec_tx_entry {
+ struct rte_mbuf *mbuf;
+};
+
struct ice_tx_queue {
uint16_t nb_tx_desc; /* number of TX descriptors */
rte_iova_t tx_ring_dma; /* TX ring DMA address */
diff --git a/drivers/net/ice/ice_rxtx_vec_avx512.c b/drivers/net/ice/ice_rxtx_vec_avx512.c
index a2a5d9987a..e5e7cc1482 100644
--- a/drivers/net/ice/ice_rxtx_vec_avx512.c
+++ b/drivers/net/ice/ice_rxtx_vec_avx512.c
@@ -756,6 +756,108 @@ ice_recv_scattered_pkts_vec_avx512(void *rx_queue, struct rte_mbuf **rx_pkts,
rx_pkts + retval, nb_pkts);
}
+static __rte_always_inline int
+ice_tx_free_bufs_avx512(struct ice_tx_queue *txq)
+{
+ struct ice_vec_tx_entry *txep;
+ uint32_t n;
+ uint32_t i;
+ int nb_free = 0;
+ struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ];
+
+ /* check DD bits on threshold descriptor */
+ if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
+ rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
+ rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
+ return 0;
+
+ n = txq->tx_rs_thresh;
+
+ /* first buffer to free from S/W ring is at index
+ * tx_next_dd - (tx_rs_thresh - 1)
+ */
+ txep = (void *)txq->sw_ring;
+ txep += txq->tx_next_dd - (n - 1);
+
+ if (txq->offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE && (n & 31) == 0) {
+ struct rte_mempool *mp = txep[0].mbuf->pool;
+ struct rte_mempool_cache *cache = rte_mempool_default_cache(mp,
+ rte_lcore_id());
+ void **cache_objs = &cache->objs[cache->len];
+
+ if (n > RTE_MEMPOOL_CACHE_MAX_SIZE) {
+ rte_mempool_ops_enqueue_bulk(mp, (void *)txep, n);
+ goto done;
+ }
+
+ /* The cache follows the following algorithm
+ * 1. Add the objects to the cache
+ * 2. Anything greater than the cache min value (if it
+ * crosses the cache flush threshold) is flushed to the ring.
+ */
+ /* Add elements back into the cache */
+ uint32_t copied = 0;
+ /* n is multiple of 32 */
+ while (copied < n) {
+ const __m512i a = _mm512_loadu_si512(&txep[copied]);
+ const __m512i b = _mm512_loadu_si512(&txep[copied + 8]);
+ const __m512i c = _mm512_loadu_si512(&txep[copied + 16]);
+ const __m512i d = _mm512_loadu_si512(&txep[copied + 24]);
+
+ _mm512_storeu_si512(&cache_objs[copied], a);
+ _mm512_storeu_si512(&cache_objs[copied + 8], b);
+ _mm512_storeu_si512(&cache_objs[copied + 16], c);
+ _mm512_storeu_si512(&cache_objs[copied + 24], d);
+ copied += 32;
+ }
+ cache->len += n;
+
+ if (cache->len >= cache->flushthresh) {
+ rte_mempool_ops_enqueue_bulk
+ (mp, &cache->objs[cache->size],
+ cache->len - cache->size);
+ cache->len = cache->size;
+ }
+ goto done;
+ }
+
+ m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
+ if (likely(m)) {
+ free[0] = m;
+ nb_free = 1;
+ for (i = 1; i < n; i++) {
+ m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
+ if (likely(m)) {
+ if (likely(m->pool == free[0]->pool)) {
+ free[nb_free++] = m;
+ } else {
+ rte_mempool_put_bulk(free[0]->pool,
+ (void *)free,
+ nb_free);
+ free[0] = m;
+ nb_free = 1;
+ }
+ }
+ }
+ rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
+ } else {
+ for (i = 1; i < n; i++) {
+ m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
+ if (m)
+ rte_mempool_put(m->pool, m);
+ }
+ }
+
+done:
+ /* buffers were freed, update counters */
+ txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
+ txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
+ if (txq->tx_next_dd >= txq->nb_tx_desc)
+ txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
+
+ return txq->tx_rs_thresh;
+}
+
static inline void
ice_vtx1(volatile struct ice_tx_desc *txdp,
struct rte_mbuf *pkt, uint64_t flags)
@@ -766,7 +868,7 @@ ice_vtx1(volatile struct ice_tx_desc *txdp,
((uint64_t)pkt->data_len << ICE_TXD_QW1_TX_BUF_SZ_S));
__m128i descriptor = _mm_set_epi64x(high_qw,
- pkt->buf_physaddr + pkt->data_off);
+ pkt->buf_iova + pkt->data_off);
_mm_store_si128((__m128i *)txdp, descriptor);
}
@@ -777,13 +879,6 @@ ice_vtx(volatile struct ice_tx_desc *txdp,
const uint64_t hi_qw_tmpl = (ICE_TX_DESC_DTYPE_DATA |
((uint64_t)flags << ICE_TXD_QW1_CMD_S));
- /* if unaligned on 32-bit boundary, do one to align */
- if (((uintptr_t)txdp & 0x1F) != 0 && nb_pkts != 0) {
- ice_vtx1(txdp, *pkt, flags);
- nb_pkts--, txdp++, pkt++;
- }
-
- /* do two at a time while possible, in bursts */
for (; nb_pkts > 3; txdp += 4, pkt += 4, nb_pkts -= 4) {
uint64_t hi_qw3 =
hi_qw_tmpl |
@@ -802,20 +897,17 @@ ice_vtx(volatile struct ice_tx_desc *txdp,
((uint64_t)pkt[0]->data_len <<
ICE_TXD_QW1_TX_BUF_SZ_S);
- __m256i desc2_3 =
- _mm256_set_epi64x
+ __m512i desc0_3 =
+ _mm512_set_epi64
(hi_qw3,
- pkt[3]->buf_physaddr + pkt[3]->data_off,
+ pkt[3]->buf_iova + pkt[3]->data_off,
hi_qw2,
- pkt[2]->buf_physaddr + pkt[2]->data_off);
- __m256i desc0_1 =
- _mm256_set_epi64x
- (hi_qw1,
- pkt[1]->buf_physaddr + pkt[1]->data_off,
+ pkt[2]->buf_iova + pkt[2]->data_off,
+ hi_qw1,
+ pkt[1]->buf_iova + pkt[1]->data_off,
hi_qw0,
- pkt[0]->buf_physaddr + pkt[0]->data_off);
- _mm256_store_si256((void *)(txdp + 2), desc2_3);
- _mm256_store_si256((void *)txdp, desc0_1);
+ pkt[0]->buf_iova + pkt[0]->data_off);
+ _mm512_storeu_si512((void *)txdp, desc0_3);
}
/* do any last ones */
@@ -825,13 +917,23 @@ ice_vtx(volatile struct ice_tx_desc *txdp,
}
}
+static __rte_always_inline void
+ice_tx_backlog_entry_avx512(struct ice_vec_tx_entry *txep,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ int i;
+
+ for (i = 0; i < (int)nb_pkts; ++i)
+ txep[i].mbuf = tx_pkts[i];
+}
+
static inline uint16_t
ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue;
volatile struct ice_tx_desc *txdp;
- struct ice_tx_entry *txep;
+ struct ice_vec_tx_entry *txep;
uint16_t n, nb_commit, tx_id;
uint64_t flags = ICE_TD_CMD;
uint64_t rs = ICE_TX_DESC_CMD_RS | ICE_TD_CMD;
@@ -840,7 +942,7 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh);
if (txq->nb_tx_free < txq->tx_free_thresh)
- ice_tx_free_bufs(txq);
+ ice_tx_free_bufs_avx512(txq);
nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
if (unlikely(nb_pkts == 0))
@@ -848,13 +950,14 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
tx_id = txq->tx_tail;
txdp = &txq->tx_ring[tx_id];
- txep = &txq->sw_ring[tx_id];
+ txep = (void *)txq->sw_ring;
+ txep += tx_id;
txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
n = (uint16_t)(txq->nb_tx_desc - tx_id);
if (nb_commit >= n) {
- ice_tx_backlog_entry(txep, tx_pkts, n);
+ ice_tx_backlog_entry_avx512(txep, tx_pkts, n);
ice_vtx(txdp, tx_pkts, n - 1, flags);
tx_pkts += (n - 1);
@@ -868,11 +971,11 @@ ice_xmit_fixed_burst_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
/* avoid reach the end of ring */
- txdp = &txq->tx_ring[tx_id];
- txep = &txq->sw_ring[tx_id];
+ txdp = txq->tx_ring;
+ txep = (void *)txq->sw_ring;
}
- ice_tx_backlog_entry(txep, tx_pkts, nb_commit);
+ ice_tx_backlog_entry_avx512(txep, tx_pkts, nb_commit);
ice_vtx(txdp, tx_pkts, nb_commit, flags);
diff --git a/drivers/net/ice/ice_rxtx_vec_common.h b/drivers/net/ice/ice_rxtx_vec_common.h
index e2019c8d6a..ae2ac29f2a 100644
--- a/drivers/net/ice/ice_rxtx_vec_common.h
+++ b/drivers/net/ice/ice_rxtx_vec_common.h
@@ -189,16 +189,38 @@ _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)
* so need to free remains more carefully.
*/
i = txq->tx_next_dd - txq->tx_rs_thresh + 1;
- if (txq->tx_tail < i) {
- for (; i < txq->nb_tx_desc; i++) {
+
+#ifdef CC_AVX512_SUPPORT
+ struct rte_eth_dev *dev = txq->vsi->adapter->eth_dev;
+
+ if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512) {
+ struct ice_vec_tx_entry *swr = (void *)txq->sw_ring;
+
+ if (txq->tx_tail < i) {
+ for (; i < txq->nb_tx_desc; i++) {
+ rte_pktmbuf_free_seg(swr[i].mbuf);
+ swr[i].mbuf = NULL;
+ }
+ i = 0;
+ }
+ for (; i < txq->tx_tail; i++) {
+ rte_pktmbuf_free_seg(swr[i].mbuf);
+ swr[i].mbuf = NULL;
+ }
+ } else
+#endif
+ {
+ if (txq->tx_tail < i) {
+ for (; i < txq->nb_tx_desc; i++) {
+ rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
+ txq->sw_ring[i].mbuf = NULL;
+ }
+ i = 0;
+ }
+ for (; i < txq->tx_tail; i++) {
rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
txq->sw_ring[i].mbuf = NULL;
}
- i = 0;
- }
- for (; i < txq->tx_tail; i++) {
- rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
- txq->sw_ring[i].mbuf = NULL;
}
}
--
2.17.1
next prev parent reply other threads:[~2020-09-18 3:54 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-09-10 6:55 [dpdk-dev] [PATCH v1 0/2] AVX512 vPMD on ice Leyi Rong
2020-09-10 6:55 ` [dpdk-dev] [PATCH v1 1/2] net/ice: add AVX512 vector path Leyi Rong
2020-09-10 9:32 ` Bruce Richardson
2020-09-10 6:55 ` [dpdk-dev] [PATCH v1 2/2] net/ice: optimize Tx path on AVX512 vPMD Leyi Rong
2020-09-15 1:17 ` Wang, Haiyue
2020-09-18 3:35 ` [dpdk-dev] [PATCH v2 0/3] AVX512 vPMD on ice Leyi Rong
2020-09-18 3:35 ` [dpdk-dev] [PATCH v2 1/3] net/ice: add AVX512 vector path Leyi Rong
2020-09-18 3:35 ` [dpdk-dev] [PATCH v2 2/3] net/ice: add RSS hash parsing in AVX512 path Leyi Rong
2020-09-18 3:35 ` Leyi Rong [this message]
2020-10-20 10:51 ` [dpdk-dev] [PATCH v3 0/3] AVX512 vPMD on ice Leyi Rong
2020-10-20 10:51 ` [dpdk-dev] [PATCH v3 1/3] net/ice: add AVX512 vector path Leyi Rong
2020-10-20 10:51 ` [dpdk-dev] [PATCH v3 2/3] net/ice: add RSS hash parsing in AVX512 path Leyi Rong
2020-10-20 10:51 ` [dpdk-dev] [PATCH v3 3/3] net/ice: optimize Tx path on AVX512 vPMD Leyi Rong
2020-10-23 4:14 ` [dpdk-dev] [PATCH v4 0/3] AVX512 vPMD on ice Leyi Rong
2020-10-23 4:14 ` [dpdk-dev] [PATCH v4 1/3] net/ice: add AVX512 vector path Leyi Rong
2020-10-25 16:23 ` David Marchand
2020-10-26 7:12 ` Rong, Leyi
2020-10-26 8:09 ` David Marchand
2020-10-27 10:19 ` Bruce Richardson
2020-10-27 10:22 ` Ferruh Yigit
2020-10-27 8:32 ` Ali Alnubani
2020-10-27 8:42 ` Ali Alnubani
2020-10-23 4:14 ` [dpdk-dev] [PATCH v4 2/3] net/ice: add RSS hash parsing in AVX512 path Leyi Rong
2020-10-23 4:14 ` [dpdk-dev] [PATCH v4 3/3] net/ice: optimize Tx path on AVX512 vPMD Leyi Rong
2020-10-23 9:39 ` [dpdk-dev] [PATCH v4 0/3] AVX512 vPMD on ice Zhang, Qi Z
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200918033528.110297-4-leyi.rong@intel.com \
--to=leyi.rong@intel.com \
--cc=bruce.richardson@intel.com \
--cc=burce.richardson@intel.com \
--cc=dev@dpdk.org \
--cc=qi.z.zhang@intel.com \
--cc=wenzhuo.lu@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).