From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 506891B47A for ; Fri, 22 Mar 2019 10:58:46 +0100 (CET) Received: from smtp.corp.redhat.com (int-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.12]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id A9F978666D; Fri, 22 Mar 2019 09:58:45 +0000 (UTC) Received: from [10.36.112.59] (ovpn-112-59.ams2.redhat.com [10.36.112.59]) by smtp.corp.redhat.com (Postfix) with ESMTPS id B41D060BE2; Fri, 22 Mar 2019 09:58:44 +0000 (UTC) To: Wenzhuo Lu , dev@dpdk.org References: <1551340136-83843-1-git-send-email-wenzhuo.lu@intel.com> <1553223516-118453-1-git-send-email-wenzhuo.lu@intel.com> <1553223516-118453-6-git-send-email-wenzhuo.lu@intel.com> From: Maxime Coquelin Message-ID: <051e8749-50e8-fae6-59f1-a7ae1c2675fd@redhat.com> Date: Fri, 22 Mar 2019 10:58:42 +0100 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Thunderbird/60.5.1 MIME-Version: 1.0 In-Reply-To: <1553223516-118453-6-git-send-email-wenzhuo.lu@intel.com> Content-Type: text/plain; charset=utf-8; format=flowed Content-Language: en-US Content-Transfer-Encoding: 7bit X-Scanned-By: MIMEDefang 2.79 on 10.5.11.12 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.26]); Fri, 22 Mar 2019 09:58:45 +0000 (UTC) Subject: Re: [dpdk-dev] [PATCH v5 5/8] net/ice: support Tx SSE vector X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 Mar 2019 09:58:46 -0000 On 3/22/19 3:58 AM, Wenzhuo Lu wrote: > Signed-off-by: Wenzhuo Lu > --- > doc/guides/nics/features/ice_vec.ini | 2 + > drivers/net/ice/ice_rxtx.c | 17 +++++ > drivers/net/ice/ice_rxtx.h | 4 + > drivers/net/ice/ice_rxtx_vec_common.h | 133 +++++++++++++++++++++++++++++++++ > drivers/net/ice/ice_rxtx_vec_sse.c | 135 ++++++++++++++++++++++++++++++++++ > 5 files changed, 291 insertions(+) > > diff --git a/doc/guides/nics/features/ice_vec.ini b/doc/guides/nics/features/ice_vec.ini > index 1a19788..173c8f2 100644 > --- a/doc/guides/nics/features/ice_vec.ini > +++ b/doc/guides/nics/features/ice_vec.ini > @@ -12,6 +12,7 @@ Queue start/stop = Y > MTU update = Y > Jumbo frame = Y > Scattered Rx = Y > +TSO = Y > Promiscuous mode = Y > Allmulticast mode = Y > Unicast MAC filter = Y > @@ -22,6 +23,7 @@ RSS reta update = Y > VLAN filter = Y > Packet type parsing = Y > Rx descriptor status = Y > +Tx descriptor status = Y > Basic stats = Y > Extended stats = Y > FW version = Y > diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c > index 5409dd0..f9ecffa 100644 > --- a/drivers/net/ice/ice_rxtx.c > +++ b/drivers/net/ice/ice_rxtx.c > @@ -2332,6 +2332,23 @@ void __attribute__((cold)) > { > struct ice_adapter *ad = > ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); > +#ifdef RTE_ARCH_X86 > + struct ice_tx_queue *txq; > + int i; > + > + if (!ice_tx_vec_dev_check(dev)) { > + for (i = 0; i < dev->data->nb_tx_queues; i++) { > + txq = dev->data->tx_queues[i]; > + (void)ice_txq_vec_setup(txq); > + } > + PMD_DRV_LOG(DEBUG, "Using Vector Tx (port %d).", > + dev->data->port_id); > + dev->tx_pkt_burst = ice_xmit_pkts_vec; > + dev->tx_pkt_prepare = NULL; > + > + return; > + } > +#endif > > if (ad->tx_simple_allowed) { > PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); > diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h > index 6ef0a84..1dde4e7 100644 > --- a/drivers/net/ice/ice_rxtx.h > +++ b/drivers/net/ice/ice_rxtx.h > @@ -170,9 +170,13 @@ void ice_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, > const uint32_t *ice_dev_supported_ptypes_get(struct rte_eth_dev *dev); > > int ice_rx_vec_dev_check(struct rte_eth_dev *dev); > +int ice_tx_vec_dev_check(struct rte_eth_dev *dev); > int ice_rxq_vec_setup(struct ice_rx_queue *rxq); > +int ice_txq_vec_setup(struct ice_tx_queue *txq); > uint16_t ice_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, > uint16_t nb_pkts); > uint16_t ice_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, > uint16_t nb_pkts); > +uint16_t ice_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, > + uint16_t nb_pkts); > #endif /* _ICE_RXTX_H_ */ > diff --git a/drivers/net/ice/ice_rxtx_vec_common.h b/drivers/net/ice/ice_rxtx_vec_common.h > index cfef91b..be079a3 100644 > --- a/drivers/net/ice/ice_rxtx_vec_common.h > +++ b/drivers/net/ice/ice_rxtx_vec_common.h > @@ -71,6 +71,73 @@ > return pkt_idx; > } > > +static __rte_always_inline int > +ice_tx_free_bufs(struct ice_tx_queue *txq) > +{ > + struct ice_tx_entry *txep; > + uint32_t n; > + uint32_t i; > + int nb_free = 0; > + struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ]; > + > + /* check DD bits on threshold descriptor */ > + if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & > + rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) != > + rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE)) > + return 0; > + > + n = txq->tx_rs_thresh; > + > + /* first buffer to free from S/W ring is at index > + * tx_next_dd - (tx_rs_thresh-1) > + */ > + txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; > + m = rte_pktmbuf_prefree_seg(txep[0].mbuf); > + if (likely(m)) { > + free[0] = m; > + nb_free = 1; > + for (i = 1; i < n; i++) { > + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); > + if (likely(m)) { > + if (likely(m->pool == free[0]->pool)) { > + free[nb_free++] = m; > + } else { > + rte_mempool_put_bulk(free[0]->pool, > + (void *)free, > + nb_free); > + free[0] = m; > + nb_free = 1; > + } > + } > + } > + rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); > + } else { > + for (i = 1; i < n; i++) { > + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); > + if (m) > + rte_mempool_put(m->pool, m); > + } > + } > + > + /* buffers were freed, update counters */ > + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); > + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); > + if (txq->tx_next_dd >= txq->nb_tx_desc) > + txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); > + > + return txq->tx_rs_thresh; > +} > + > +static __rte_always_inline void > +tx_backlog_entry(struct ice_tx_entry *txep, Consider prefixing it with 'ice_tx_'. > + struct rte_mbuf **tx_pkts, uint16_t nb_pkts) > +{ > + int i; > + > + for (i = 0; i < (int)nb_pkts; ++i) > + txep[i].mbuf = tx_pkts[i]; > +} > + > static inline void > _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) > { > @@ -101,6 +168,34 @@ > memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); > } > > +static inline void > +_ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) > +{ > + uint16_t i; > + > + if (!txq || !txq->sw_ring) { if (unlikely(...)) { > + PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); s/rxq/txq/ > + return; > + } > + > + /** > + * vPMD tx will not set sw_ring's mbuf to NULL after free, > + * so need to free remains more carefully. > + */ > + i = txq->tx_next_dd - txq->tx_rs_thresh + 1; > + if (txq->tx_tail < i) { > + for (; i < txq->nb_tx_desc; i++) { > + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); > + txq->sw_ring[i].mbuf = NULL; > + } > + i = 0; > + } > + for (; i < txq->tx_tail; i++) { > + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); > + txq->sw_ring[i].mbuf = NULL; > + } > +} > + > static inline int > ice_rxq_vec_setup_default(struct ice_rx_queue *rxq) > { > @@ -137,6 +232,29 @@ > return 0; > } > > +#define ICE_NO_VECTOR_FLAGS ( \ > + DEV_TX_OFFLOAD_MULTI_SEGS | \ > + DEV_TX_OFFLOAD_VLAN_INSERT | \ > + DEV_TX_OFFLOAD_SCTP_CKSUM | \ > + DEV_TX_OFFLOAD_UDP_CKSUM | \ > + DEV_TX_OFFLOAD_TCP_CKSUM) > + > +static inline int > +ice_tx_vec_queue_default(struct ice_tx_queue *txq) > +{ > + if (!txq) > + return -1; > + > + if (txq->offloads & ICE_NO_VECTOR_FLAGS) > + return -1; > + > + if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST || > + txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ) > + return -1; > + > + return 0; > +} > + > static inline int > ice_rx_vec_dev_check_default(struct rte_eth_dev *dev) > { > @@ -152,4 +270,19 @@ > return 0; > } > > +static inline int > +ice_tx_vec_dev_check_default(struct rte_eth_dev *dev) > +{ > + int i; > + struct ice_tx_queue *txq; > + > + for (i = 0; i < dev->data->nb_tx_queues; i++) { > + txq = dev->data->tx_queues[i]; > + if (ice_tx_vec_queue_default(txq)) > + return -1; return ice_tx_vec_queue_default(txq); Applies also to rx path. > + } > + > + return 0; > +} > + > #endif > diff --git a/drivers/net/ice/ice_rxtx_vec_sse.c b/drivers/net/ice/ice_rxtx_vec_sse.c > index e1f057a..4e148d6 100644 > --- a/drivers/net/ice/ice_rxtx_vec_sse.c > +++ b/drivers/net/ice/ice_rxtx_vec_sse.c > @@ -514,12 +514,131 @@ > &split_flags[i]); > } > > +static inline void > +ice_vtx1(volatile struct ice_tx_desc *txdp, struct rte_mbuf *pkt, > + uint64_t flags) > +{ > + uint64_t high_qw = > + (ICE_TX_DESC_DTYPE_DATA | > + ((uint64_t)flags << ICE_TXD_QW1_CMD_S) | > + ((uint64_t)pkt->data_len << ICE_TXD_QW1_TX_BUF_SZ_S)); > + > + __m128i descriptor = _mm_set_epi64x(high_qw, > + pkt->buf_iova + pkt->data_off); > + _mm_store_si128((__m128i *)txdp, descriptor); > +} > + > +static inline void > +ice_vtx(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkt, > + uint16_t nb_pkts, uint64_t flags) > +{ > + int i; > + > + for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) > + ice_vtx1(txdp, *pkt, flags); > +} > + > +static uint16_t > +ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, > + uint16_t nb_pkts) > +{ > + struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; > + volatile struct ice_tx_desc *txdp; > + struct ice_tx_entry *txep; > + uint16_t n, nb_commit, tx_id; > + uint64_t flags = ICE_TD_CMD; > + uint64_t rs = ICE_TX_DESC_CMD_RS | ICE_TD_CMD; > + int i; > + > + /* cross rx_thresh boundary is not allowed */ > + nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); > + > + if (txq->nb_tx_free < txq->tx_free_thresh) > + ice_tx_free_bufs(txq); > + > + nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); > + nb_commit = nb_pkts; > + if (unlikely(nb_pkts == 0)) > + return 0; > + > + tx_id = txq->tx_tail; > + txdp = &txq->tx_ring[tx_id]; > + txep = &txq->sw_ring[tx_id]; > + > + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); > + > + n = (uint16_t)(txq->nb_tx_desc - tx_id); > + if (nb_commit >= n) { > + tx_backlog_entry(txep, tx_pkts, n); > + > + for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) > + ice_vtx1(txdp, *tx_pkts, flags); > + > + ice_vtx1(txdp, *tx_pkts++, rs); > + > + nb_commit = (uint16_t)(nb_commit - n); > + > + tx_id = 0; > + txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); > + > + /* avoid reach the end of ring */ > + txdp = &txq->tx_ring[tx_id]; > + txep = &txq->sw_ring[tx_id]; > + } > + > + tx_backlog_entry(txep, tx_pkts, nb_commit); > + > + ice_vtx(txdp, tx_pkts, nb_commit, flags); > + > + tx_id = (uint16_t)(tx_id + nb_commit); > + if (tx_id > txq->tx_next_rs) { > + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= > + rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) << > + ICE_TXD_QW1_CMD_S); > + txq->tx_next_rs = > + (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); > + } > + > + txq->tx_tail = tx_id; > + > + ICE_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); > + > + return nb_pkts; > +} > + > +uint16_t > +ice_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, > + uint16_t nb_pkts) > +{ > + uint16_t nb_tx = 0; > + struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; > + > + while (nb_pkts) { > + uint16_t ret, num; > + > + num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh); > + ret = ice_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], num); > + nb_tx += ret; > + nb_pkts -= ret; > + if (ret < num) > + break; > + } > + > + return nb_tx; > +} > + > static void __attribute__((cold)) > ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) > { > _ice_rx_queue_release_mbufs_vec(rxq); > } > > +static void __attribute__((cold)) > +ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) > +{ > + _ice_tx_queue_release_mbufs_vec(txq); As for Rx, consider putting the code firectly here as _ice_tx_queue_release_mbufs_vec() is called only once here. > +} > + > int __attribute__((cold)) > ice_rxq_vec_setup(struct ice_rx_queue *rxq) > { > @@ -531,7 +650,23 @@ int __attribute__((cold)) > } > > int __attribute__((cold)) > +ice_txq_vec_setup(struct ice_tx_queue __rte_unused *txq) > +{ > + if (!txq) > + return -1; > + > + txq->tx_rel_mbufs = ice_tx_queue_release_mbufs_vec; > + return 0; > +} > + > +int __attribute__((cold)) > ice_rx_vec_dev_check(struct rte_eth_dev *dev) > { > return ice_rx_vec_dev_check_default(dev); > } > + > +int __attribute__((cold)) > +ice_tx_vec_dev_check(struct rte_eth_dev *dev) > +{ > + return ice_tx_vec_dev_check_default(dev); > +} > From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by dpdk.space (Postfix) with ESMTP id 888C3A00E6 for ; Fri, 22 Mar 2019 10:58:49 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id A41DF1B596; Fri, 22 Mar 2019 10:58:47 +0100 (CET) Received: from mx1.redhat.com (mx1.redhat.com [209.132.183.28]) by dpdk.org (Postfix) with ESMTP id 506891B47A for ; Fri, 22 Mar 2019 10:58:46 +0100 (CET) Received: from smtp.corp.redhat.com (int-mx02.intmail.prod.int.phx2.redhat.com [10.5.11.12]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id A9F978666D; Fri, 22 Mar 2019 09:58:45 +0000 (UTC) Received: from [10.36.112.59] (ovpn-112-59.ams2.redhat.com [10.36.112.59]) by smtp.corp.redhat.com (Postfix) with ESMTPS id B41D060BE2; Fri, 22 Mar 2019 09:58:44 +0000 (UTC) To: Wenzhuo Lu , dev@dpdk.org References: <1551340136-83843-1-git-send-email-wenzhuo.lu@intel.com> <1553223516-118453-1-git-send-email-wenzhuo.lu@intel.com> <1553223516-118453-6-git-send-email-wenzhuo.lu@intel.com> From: Maxime Coquelin Message-ID: <051e8749-50e8-fae6-59f1-a7ae1c2675fd@redhat.com> Date: Fri, 22 Mar 2019 10:58:42 +0100 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Thunderbird/60.5.1 MIME-Version: 1.0 In-Reply-To: <1553223516-118453-6-git-send-email-wenzhuo.lu@intel.com> Content-Type: text/plain; charset="UTF-8"; format="flowed" Content-Language: en-US Content-Transfer-Encoding: 7bit X-Scanned-By: MIMEDefang 2.79 on 10.5.11.12 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.26]); Fri, 22 Mar 2019 09:58:45 +0000 (UTC) Subject: Re: [dpdk-dev] [PATCH v5 5/8] net/ice: support Tx SSE vector X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Message-ID: <20190322095842.h9q69wfpaSoQj-wKhXes8szsnILigO1wTLBPF35_qDw@z> On 3/22/19 3:58 AM, Wenzhuo Lu wrote: > Signed-off-by: Wenzhuo Lu > --- > doc/guides/nics/features/ice_vec.ini | 2 + > drivers/net/ice/ice_rxtx.c | 17 +++++ > drivers/net/ice/ice_rxtx.h | 4 + > drivers/net/ice/ice_rxtx_vec_common.h | 133 +++++++++++++++++++++++++++++++++ > drivers/net/ice/ice_rxtx_vec_sse.c | 135 ++++++++++++++++++++++++++++++++++ > 5 files changed, 291 insertions(+) > > diff --git a/doc/guides/nics/features/ice_vec.ini b/doc/guides/nics/features/ice_vec.ini > index 1a19788..173c8f2 100644 > --- a/doc/guides/nics/features/ice_vec.ini > +++ b/doc/guides/nics/features/ice_vec.ini > @@ -12,6 +12,7 @@ Queue start/stop = Y > MTU update = Y > Jumbo frame = Y > Scattered Rx = Y > +TSO = Y > Promiscuous mode = Y > Allmulticast mode = Y > Unicast MAC filter = Y > @@ -22,6 +23,7 @@ RSS reta update = Y > VLAN filter = Y > Packet type parsing = Y > Rx descriptor status = Y > +Tx descriptor status = Y > Basic stats = Y > Extended stats = Y > FW version = Y > diff --git a/drivers/net/ice/ice_rxtx.c b/drivers/net/ice/ice_rxtx.c > index 5409dd0..f9ecffa 100644 > --- a/drivers/net/ice/ice_rxtx.c > +++ b/drivers/net/ice/ice_rxtx.c > @@ -2332,6 +2332,23 @@ void __attribute__((cold)) > { > struct ice_adapter *ad = > ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); > +#ifdef RTE_ARCH_X86 > + struct ice_tx_queue *txq; > + int i; > + > + if (!ice_tx_vec_dev_check(dev)) { > + for (i = 0; i < dev->data->nb_tx_queues; i++) { > + txq = dev->data->tx_queues[i]; > + (void)ice_txq_vec_setup(txq); > + } > + PMD_DRV_LOG(DEBUG, "Using Vector Tx (port %d).", > + dev->data->port_id); > + dev->tx_pkt_burst = ice_xmit_pkts_vec; > + dev->tx_pkt_prepare = NULL; > + > + return; > + } > +#endif > > if (ad->tx_simple_allowed) { > PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); > diff --git a/drivers/net/ice/ice_rxtx.h b/drivers/net/ice/ice_rxtx.h > index 6ef0a84..1dde4e7 100644 > --- a/drivers/net/ice/ice_rxtx.h > +++ b/drivers/net/ice/ice_rxtx.h > @@ -170,9 +170,13 @@ void ice_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, > const uint32_t *ice_dev_supported_ptypes_get(struct rte_eth_dev *dev); > > int ice_rx_vec_dev_check(struct rte_eth_dev *dev); > +int ice_tx_vec_dev_check(struct rte_eth_dev *dev); > int ice_rxq_vec_setup(struct ice_rx_queue *rxq); > +int ice_txq_vec_setup(struct ice_tx_queue *txq); > uint16_t ice_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, > uint16_t nb_pkts); > uint16_t ice_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, > uint16_t nb_pkts); > +uint16_t ice_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, > + uint16_t nb_pkts); > #endif /* _ICE_RXTX_H_ */ > diff --git a/drivers/net/ice/ice_rxtx_vec_common.h b/drivers/net/ice/ice_rxtx_vec_common.h > index cfef91b..be079a3 100644 > --- a/drivers/net/ice/ice_rxtx_vec_common.h > +++ b/drivers/net/ice/ice_rxtx_vec_common.h > @@ -71,6 +71,73 @@ > return pkt_idx; > } > > +static __rte_always_inline int > +ice_tx_free_bufs(struct ice_tx_queue *txq) > +{ > + struct ice_tx_entry *txep; > + uint32_t n; > + uint32_t i; > + int nb_free = 0; > + struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ]; > + > + /* check DD bits on threshold descriptor */ > + if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & > + rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) != > + rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE)) > + return 0; > + > + n = txq->tx_rs_thresh; > + > + /* first buffer to free from S/W ring is at index > + * tx_next_dd - (tx_rs_thresh-1) > + */ > + txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; > + m = rte_pktmbuf_prefree_seg(txep[0].mbuf); > + if (likely(m)) { > + free[0] = m; > + nb_free = 1; > + for (i = 1; i < n; i++) { > + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); > + if (likely(m)) { > + if (likely(m->pool == free[0]->pool)) { > + free[nb_free++] = m; > + } else { > + rte_mempool_put_bulk(free[0]->pool, > + (void *)free, > + nb_free); > + free[0] = m; > + nb_free = 1; > + } > + } > + } > + rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); > + } else { > + for (i = 1; i < n; i++) { > + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); > + if (m) > + rte_mempool_put(m->pool, m); > + } > + } > + > + /* buffers were freed, update counters */ > + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); > + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); > + if (txq->tx_next_dd >= txq->nb_tx_desc) > + txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); > + > + return txq->tx_rs_thresh; > +} > + > +static __rte_always_inline void > +tx_backlog_entry(struct ice_tx_entry *txep, Consider prefixing it with 'ice_tx_'. > + struct rte_mbuf **tx_pkts, uint16_t nb_pkts) > +{ > + int i; > + > + for (i = 0; i < (int)nb_pkts; ++i) > + txep[i].mbuf = tx_pkts[i]; > +} > + > static inline void > _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) > { > @@ -101,6 +168,34 @@ > memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); > } > > +static inline void > +_ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) > +{ > + uint16_t i; > + > + if (!txq || !txq->sw_ring) { if (unlikely(...)) { > + PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); s/rxq/txq/ > + return; > + } > + > + /** > + * vPMD tx will not set sw_ring's mbuf to NULL after free, > + * so need to free remains more carefully. > + */ > + i = txq->tx_next_dd - txq->tx_rs_thresh + 1; > + if (txq->tx_tail < i) { > + for (; i < txq->nb_tx_desc; i++) { > + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); > + txq->sw_ring[i].mbuf = NULL; > + } > + i = 0; > + } > + for (; i < txq->tx_tail; i++) { > + rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); > + txq->sw_ring[i].mbuf = NULL; > + } > +} > + > static inline int > ice_rxq_vec_setup_default(struct ice_rx_queue *rxq) > { > @@ -137,6 +232,29 @@ > return 0; > } > > +#define ICE_NO_VECTOR_FLAGS ( \ > + DEV_TX_OFFLOAD_MULTI_SEGS | \ > + DEV_TX_OFFLOAD_VLAN_INSERT | \ > + DEV_TX_OFFLOAD_SCTP_CKSUM | \ > + DEV_TX_OFFLOAD_UDP_CKSUM | \ > + DEV_TX_OFFLOAD_TCP_CKSUM) > + > +static inline int > +ice_tx_vec_queue_default(struct ice_tx_queue *txq) > +{ > + if (!txq) > + return -1; > + > + if (txq->offloads & ICE_NO_VECTOR_FLAGS) > + return -1; > + > + if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST || > + txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ) > + return -1; > + > + return 0; > +} > + > static inline int > ice_rx_vec_dev_check_default(struct rte_eth_dev *dev) > { > @@ -152,4 +270,19 @@ > return 0; > } > > +static inline int > +ice_tx_vec_dev_check_default(struct rte_eth_dev *dev) > +{ > + int i; > + struct ice_tx_queue *txq; > + > + for (i = 0; i < dev->data->nb_tx_queues; i++) { > + txq = dev->data->tx_queues[i]; > + if (ice_tx_vec_queue_default(txq)) > + return -1; return ice_tx_vec_queue_default(txq); Applies also to rx path. > + } > + > + return 0; > +} > + > #endif > diff --git a/drivers/net/ice/ice_rxtx_vec_sse.c b/drivers/net/ice/ice_rxtx_vec_sse.c > index e1f057a..4e148d6 100644 > --- a/drivers/net/ice/ice_rxtx_vec_sse.c > +++ b/drivers/net/ice/ice_rxtx_vec_sse.c > @@ -514,12 +514,131 @@ > &split_flags[i]); > } > > +static inline void > +ice_vtx1(volatile struct ice_tx_desc *txdp, struct rte_mbuf *pkt, > + uint64_t flags) > +{ > + uint64_t high_qw = > + (ICE_TX_DESC_DTYPE_DATA | > + ((uint64_t)flags << ICE_TXD_QW1_CMD_S) | > + ((uint64_t)pkt->data_len << ICE_TXD_QW1_TX_BUF_SZ_S)); > + > + __m128i descriptor = _mm_set_epi64x(high_qw, > + pkt->buf_iova + pkt->data_off); > + _mm_store_si128((__m128i *)txdp, descriptor); > +} > + > +static inline void > +ice_vtx(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkt, > + uint16_t nb_pkts, uint64_t flags) > +{ > + int i; > + > + for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt) > + ice_vtx1(txdp, *pkt, flags); > +} > + > +static uint16_t > +ice_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, > + uint16_t nb_pkts) > +{ > + struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; > + volatile struct ice_tx_desc *txdp; > + struct ice_tx_entry *txep; > + uint16_t n, nb_commit, tx_id; > + uint64_t flags = ICE_TD_CMD; > + uint64_t rs = ICE_TX_DESC_CMD_RS | ICE_TD_CMD; > + int i; > + > + /* cross rx_thresh boundary is not allowed */ > + nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh); > + > + if (txq->nb_tx_free < txq->tx_free_thresh) > + ice_tx_free_bufs(txq); > + > + nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); > + nb_commit = nb_pkts; > + if (unlikely(nb_pkts == 0)) > + return 0; > + > + tx_id = txq->tx_tail; > + txdp = &txq->tx_ring[tx_id]; > + txep = &txq->sw_ring[tx_id]; > + > + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); > + > + n = (uint16_t)(txq->nb_tx_desc - tx_id); > + if (nb_commit >= n) { > + tx_backlog_entry(txep, tx_pkts, n); > + > + for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) > + ice_vtx1(txdp, *tx_pkts, flags); > + > + ice_vtx1(txdp, *tx_pkts++, rs); > + > + nb_commit = (uint16_t)(nb_commit - n); > + > + tx_id = 0; > + txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); > + > + /* avoid reach the end of ring */ > + txdp = &txq->tx_ring[tx_id]; > + txep = &txq->sw_ring[tx_id]; > + } > + > + tx_backlog_entry(txep, tx_pkts, nb_commit); > + > + ice_vtx(txdp, tx_pkts, nb_commit, flags); > + > + tx_id = (uint16_t)(tx_id + nb_commit); > + if (tx_id > txq->tx_next_rs) { > + txq->tx_ring[txq->tx_next_rs].cmd_type_offset_bsz |= > + rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) << > + ICE_TXD_QW1_CMD_S); > + txq->tx_next_rs = > + (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); > + } > + > + txq->tx_tail = tx_id; > + > + ICE_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); > + > + return nb_pkts; > +} > + > +uint16_t > +ice_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, > + uint16_t nb_pkts) > +{ > + uint16_t nb_tx = 0; > + struct ice_tx_queue *txq = (struct ice_tx_queue *)tx_queue; > + > + while (nb_pkts) { > + uint16_t ret, num; > + > + num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh); > + ret = ice_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], num); > + nb_tx += ret; > + nb_pkts -= ret; > + if (ret < num) > + break; > + } > + > + return nb_tx; > +} > + > static void __attribute__((cold)) > ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) > { > _ice_rx_queue_release_mbufs_vec(rxq); > } > > +static void __attribute__((cold)) > +ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) > +{ > + _ice_tx_queue_release_mbufs_vec(txq); As for Rx, consider putting the code firectly here as _ice_tx_queue_release_mbufs_vec() is called only once here. > +} > + > int __attribute__((cold)) > ice_rxq_vec_setup(struct ice_rx_queue *rxq) > { > @@ -531,7 +650,23 @@ int __attribute__((cold)) > } > > int __attribute__((cold)) > +ice_txq_vec_setup(struct ice_tx_queue __rte_unused *txq) > +{ > + if (!txq) > + return -1; > + > + txq->tx_rel_mbufs = ice_tx_queue_release_mbufs_vec; > + return 0; > +} > + > +int __attribute__((cold)) > ice_rx_vec_dev_check(struct rte_eth_dev *dev) > { > return ice_rx_vec_dev_check_default(dev); > } > + > +int __attribute__((cold)) > +ice_tx_vec_dev_check(struct rte_eth_dev *dev) > +{ > + return ice_tx_vec_dev_check_default(dev); > +} >