From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 39880A0524; Wed, 2 Jun 2021 11:42:04 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 854884118B; Wed, 2 Jun 2021 11:40:07 +0200 (CEST) Received: from smtpbg516.qq.com (smtpbg516.qq.com [203.205.250.54]) by mails.dpdk.org (Postfix) with ESMTP id 5A19D41160 for ; Wed, 2 Jun 2021 11:40:04 +0200 (CEST) X-QQ-mid: bizesmtp32t1622626799tydzhdpe Received: from wxdbg.localdomain.com (unknown [183.129.236.74]) by esmtp6.qq.com (ESMTP) with id ; Wed, 02 Jun 2021 17:39:59 +0800 (CST) X-QQ-SSF: 01400000000000D0E000B00A0000000 X-QQ-FEAT: fYJ0eR3mMJ2wnM3AwD7cupw1bWK9QObFCqtitNkppWnRwulZhGy9EiTOpkSA3 pIWz9xSt2t0lhsKuidmtkDCI2RVL8bpQCw7n48t6iQOaflnlrJubZ/WSb66Z+4iihSV1YSe i7g+4KSlJ6u+NcxlGJdnz3L2feidSReSxthSfvxgRi2p7wjs2Ps8To0JNXZ22kbMwabfR95 2txkV1eI9QBCkqf/GODJYlzwYVy84U9W3iW9b2+pLggFCBisddwwXNOWOQLHoMhx4fyX6Jh ei/e7aEt3m//wtq0mTZpQnGdiJvcbCK0qNAqAcZBkvcOFsF4iOAVCKGjF4BzGrQcd5jfHjw mUEP1lOWuXvloVxLqauG6NP5K6big== X-QQ-GoodBg: 2 From: Jiawen Wu To: dev@dpdk.org Cc: Jiawen Wu Date: Wed, 2 Jun 2021 17:41:03 +0800 Message-Id: <20210602094108.1575640-20-jiawenwu@trustnetic.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20210602094108.1575640-1-jiawenwu@trustnetic.com> References: <20210602094108.1575640-1-jiawenwu@trustnetic.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-QQ-SENDSIZE: 520 Feedback-ID: bizesmtp:trustnetic.com:qybgforeign:qybgforeign1 X-QQ-Bgrelay: 1 Subject: [dpdk-dev] [PATCH v5 19/24] net/ngbe: add simple Rx and Tx flow X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Initialize device with the simplest receive and transmit functions. Signed-off-by: Jiawen Wu --- drivers/net/ngbe/ngbe_ethdev.c | 8 +- drivers/net/ngbe/ngbe_ethdev.h | 6 + drivers/net/ngbe/ngbe_rxtx.c | 482 +++++++++++++++++++++++++++++++++ drivers/net/ngbe/ngbe_rxtx.h | 110 ++++++++ 4 files changed, 604 insertions(+), 2 deletions(-) diff --git a/drivers/net/ngbe/ngbe_ethdev.c b/drivers/net/ngbe/ngbe_ethdev.c index 672db88133..4dab920caa 100644 --- a/drivers/net/ngbe/ngbe_ethdev.c +++ b/drivers/net/ngbe/ngbe_ethdev.c @@ -109,6 +109,8 @@ eth_ngbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused) PMD_INIT_FUNC_TRACE(); eth_dev->dev_ops = &ngbe_eth_dev_ops; + eth_dev->rx_pkt_burst = &ngbe_recv_pkts; + eth_dev->tx_pkt_burst = &ngbe_xmit_pkts_simple; if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; @@ -357,8 +359,10 @@ ngbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) const uint32_t * ngbe_dev_supported_ptypes_get(struct rte_eth_dev *dev) { - RTE_SET_USED(dev); - return ngbe_get_supported_ptypes(); + if (dev->rx_pkt_burst == ngbe_recv_pkts) + return ngbe_get_supported_ptypes(); + + return NULL; } /* return 0 means link status changed, -1 means not changed */ diff --git a/drivers/net/ngbe/ngbe_ethdev.h b/drivers/net/ngbe/ngbe_ethdev.h index 6881351252..c0f8483eca 100644 --- a/drivers/net/ngbe/ngbe_ethdev.h +++ b/drivers/net/ngbe/ngbe_ethdev.h @@ -75,6 +75,12 @@ int ngbe_dev_rx_init(struct rte_eth_dev *dev); void ngbe_dev_tx_init(struct rte_eth_dev *dev); +uint16_t ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); + +uint16_t ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts); + int ngbe_dev_link_update_share(struct rte_eth_dev *dev, int wait_to_complete); diff --git a/drivers/net/ngbe/ngbe_rxtx.c b/drivers/net/ngbe/ngbe_rxtx.c index 68d7e651af..9462da5b7a 100644 --- a/drivers/net/ngbe/ngbe_rxtx.c +++ b/drivers/net/ngbe/ngbe_rxtx.c @@ -15,10 +15,492 @@ #include "ngbe_ethdev.h" #include "ngbe_rxtx.h" +/* + * Prefetch a cache line into all cache levels. + */ +#define rte_ngbe_prefetch(p) rte_prefetch0(p) + +/********************************************************************* + * + * TX functions + * + **********************************************************************/ + +/* + * Check for descriptors with their DD bit set and free mbufs. + * Return the total number of buffers freed. + */ +static __rte_always_inline int +ngbe_tx_free_bufs(struct ngbe_tx_queue *txq) +{ + struct ngbe_tx_entry *txep; + uint32_t status; + int i, nb_free = 0; + struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ]; + + /* check DD bit on threshold descriptor */ + status = txq->tx_ring[txq->tx_next_dd].dw3; + if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { + if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) + ngbe_set32_masked(txq->tdc_reg_addr, + NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); + return 0; + } + + /* + * first buffer to free from S/W ring is at index + * tx_next_dd - (tx_free_thresh-1) + */ + txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)]; + for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) { + /* free buffers one at a time */ + m = rte_pktmbuf_prefree_seg(txep->mbuf); + txep->mbuf = NULL; + + if (unlikely(m == NULL)) + continue; + + if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ || + (nb_free > 0 && m->pool != free[0]->pool)) { + rte_mempool_put_bulk(free[0]->pool, + (void **)free, nb_free); + nb_free = 0; + } + + free[nb_free++] = m; + } + + if (nb_free > 0) + rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); + + /* buffers were freed, update counters */ + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh); + txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh); + if (txq->tx_next_dd >= txq->nb_tx_desc) + txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); + + return txq->tx_free_thresh; +} + +/* Populate 4 descriptors with data from 4 mbufs */ +static inline void +tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) +{ + uint64_t buf_dma_addr; + uint32_t pkt_len; + int i; + + for (i = 0; i < 4; ++i, ++txdp, ++pkts) { + buf_dma_addr = rte_mbuf_data_iova(*pkts); + pkt_len = (*pkts)->data_len; + + /* write data to descriptor */ + txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); + txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | + NGBE_TXD_DATLEN(pkt_len)); + txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); + + rte_prefetch0(&(*pkts)->pool); + } +} + +/* Populate 1 descriptor with data from 1 mbuf */ +static inline void +tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) +{ + uint64_t buf_dma_addr; + uint32_t pkt_len; + + buf_dma_addr = rte_mbuf_data_iova(*pkts); + pkt_len = (*pkts)->data_len; + + /* write data to descriptor */ + txdp->qw0 = cpu_to_le64(buf_dma_addr); + txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | + NGBE_TXD_DATLEN(pkt_len)); + txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); + + rte_prefetch0(&(*pkts)->pool); +} + +/* + * Fill H/W descriptor ring with mbuf data. + * Copy mbuf pointers to the S/W ring. + */ +static inline void +ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts, + uint16_t nb_pkts) +{ + volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; + struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; + const int N_PER_LOOP = 4; + const int N_PER_LOOP_MASK = N_PER_LOOP - 1; + int mainpart, leftover; + int i, j; + + /* + * Process most of the packets in chunks of N pkts. Any + * leftover packets will get processed one at a time. + */ + mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK)); + leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK)); + for (i = 0; i < mainpart; i += N_PER_LOOP) { + /* Copy N mbuf pointers to the S/W ring */ + for (j = 0; j < N_PER_LOOP; ++j) + (txep + i + j)->mbuf = *(pkts + i + j); + tx4(txdp + i, pkts + i); + } + + if (unlikely(leftover > 0)) { + for (i = 0; i < leftover; ++i) { + (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); + tx1(txdp + mainpart + i, pkts + mainpart + i); + } + } +} + +static inline uint16_t +tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; + uint16_t n = 0; + + /* + * Begin scanning the H/W ring for done descriptors when the + * number of available descriptors drops below tx_free_thresh. For + * each done descriptor, free the associated buffer. + */ + if (txq->nb_tx_free < txq->tx_free_thresh) + ngbe_tx_free_bufs(txq); + + /* Only use descriptors that are available */ + nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); + if (unlikely(nb_pkts == 0)) + return 0; + + /* Use exactly nb_pkts descriptors */ + txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); + + /* + * At this point, we know there are enough descriptors in the + * ring to transmit all the packets. This assumes that each + * mbuf contains a single segment, and that no new offloads + * are expected, which would require a new context descriptor. + */ + + /* + * See if we're going to wrap-around. If so, handle the top + * of the descriptor ring first, then do the bottom. If not, + * the processing looks just like the "bottom" part anyway... + */ + if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { + n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); + ngbe_tx_fill_hw_ring(txq, tx_pkts, n); + txq->tx_tail = 0; + } + + /* Fill H/W descriptor ring with mbuf data */ + ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); + txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); + + /* + * Check for wrap-around. This would only happen if we used + * up to the last descriptor in the ring, no more, no less. + */ + if (txq->tx_tail >= txq->nb_tx_desc) + txq->tx_tail = 0; + + PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", + (uint16_t)txq->port_id, (uint16_t)txq->queue_id, + (uint16_t)txq->tx_tail, (uint16_t)nb_pkts); + + /* update tail pointer */ + rte_wmb(); + ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail); + + return nb_pkts; +} + +uint16_t +ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + uint16_t nb_tx; + + /* Try to transmit at least chunks of TX_MAX_BURST pkts */ + if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST)) + return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); + + /* transmit more than the max burst, in chunks of TX_MAX_BURST */ + nb_tx = 0; + while (nb_pkts) { + uint16_t ret, n; + + n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST); + ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n); + nb_tx = (uint16_t)(nb_tx + ret); + nb_pkts = (uint16_t)(nb_pkts - ret); + if (ret < n) + break; + } + + return nb_tx; +} + #ifndef DEFAULT_TX_FREE_THRESH #define DEFAULT_TX_FREE_THRESH 32 #endif +/********************************************************************* + * + * RX functions + * + **********************************************************************/ +static inline uint32_t +ngbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask) +{ + uint16_t ptid = NGBE_RXD_PTID(pkt_info); + + ptid &= ptid_mask; + + return ngbe_decode_ptype(ptid); +} + +static inline uint64_t +ngbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info) +{ + static uint64_t ip_rss_types_map[16] __rte_cache_aligned = { + 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, + 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, + PKT_RX_RSS_HASH, 0, 0, 0, + 0, 0, 0, PKT_RX_FDIR, + }; + return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; +} + +static inline uint64_t +rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) +{ + uint64_t pkt_flags; + + /* + * Check if VLAN present only. + * Do not check whether L3/L4 rx checksum done by NIC or not, + * That can be found from rte_eth_rxmode.offloads flag + */ + pkt_flags = (rx_status & NGBE_RXD_STAT_VLAN && + vlan_flags & PKT_RX_VLAN_STRIPPED) + ? vlan_flags : 0; + + return pkt_flags; +} + +static inline uint64_t +rx_desc_error_to_pkt_flags(uint32_t rx_status) +{ + uint64_t pkt_flags = 0; + + /* checksum offload can't be disabled */ + if (rx_status & NGBE_RXD_STAT_IPCS) { + pkt_flags |= (rx_status & NGBE_RXD_ERR_IPCS + ? PKT_RX_IP_CKSUM_BAD : PKT_RX_IP_CKSUM_GOOD); + } + + if (rx_status & NGBE_RXD_STAT_L4CS) { + pkt_flags |= (rx_status & NGBE_RXD_ERR_L4CS + ? PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD); + } + + if (rx_status & NGBE_RXD_STAT_EIPCS && + rx_status & NGBE_RXD_ERR_EIPCS) { + pkt_flags |= PKT_RX_OUTER_IP_CKSUM_BAD; + } + + + return pkt_flags; +} + +uint16_t +ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + struct ngbe_rx_queue *rxq; + volatile struct ngbe_rx_desc *rx_ring; + volatile struct ngbe_rx_desc *rxdp; + struct ngbe_rx_entry *sw_ring; + struct ngbe_rx_entry *rxe; + struct rte_mbuf *rxm; + struct rte_mbuf *nmb; + struct ngbe_rx_desc rxd; + uint64_t dma_addr; + uint32_t staterr; + uint32_t pkt_info; + uint16_t pkt_len; + uint16_t rx_id; + uint16_t nb_rx; + uint16_t nb_hold; + uint64_t pkt_flags; + + nb_rx = 0; + nb_hold = 0; + rxq = rx_queue; + rx_id = rxq->rx_tail; + rx_ring = rxq->rx_ring; + sw_ring = rxq->sw_ring; + struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; + while (nb_rx < nb_pkts) { + /* + * The order of operations here is important as the DD status + * bit must not be read after any other descriptor fields. + * rx_ring and rxdp are pointing to volatile data so the order + * of accesses cannot be reordered by the compiler. If they were + * not volatile, they could be reordered which could lead to + * using invalid descriptor fields when read from rxd. + */ + rxdp = &rx_ring[rx_id]; + staterr = rxdp->qw1.lo.status; + if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) + break; + rxd = *rxdp; + + /* + * End of packet. + * + * If the NGBE_RXD_STAT_EOP flag is not set, the RX packet + * is likely to be invalid and to be dropped by the various + * validation checks performed by the network stack. + * + * Allocate a new mbuf to replenish the RX ring descriptor. + * If the allocation fails: + * - arrange for that RX descriptor to be the first one + * being parsed the next time the receive function is + * invoked [on the same queue]. + * + * - Stop parsing the RX ring and return immediately. + * + * This policy do not drop the packet received in the RX + * descriptor for which the allocation of a new mbuf failed. + * Thus, it allows that packet to be later retrieved if + * mbuf have been freed in the mean time. + * As a side effect, holding RX descriptors instead of + * systematically giving them back to the NIC may lead to + * RX ring exhaustion situations. + * However, the NIC can gracefully prevent such situations + * to happen by sending specific "back-pressure" flow control + * frames to its peer(s). + */ + PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " + "ext_err_stat=0x%08x pkt_len=%u", + (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, + (uint16_t)rx_id, (uint32_t)staterr, + (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len)); + + nmb = rte_mbuf_raw_alloc(rxq->mb_pool); + if (nmb == NULL) { + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " + "queue_id=%u", (uint16_t)rxq->port_id, + (uint16_t)rxq->queue_id); + dev->data->rx_mbuf_alloc_failed++; + break; + } + + nb_hold++; + rxe = &sw_ring[rx_id]; + rx_id++; + if (rx_id == rxq->nb_rx_desc) + rx_id = 0; + + /* Prefetch next mbuf while processing current one. */ + rte_ngbe_prefetch(sw_ring[rx_id].mbuf); + + /* + * When next RX descriptor is on a cache-line boundary, + * prefetch the next 4 RX descriptors and the next 8 pointers + * to mbufs. + */ + if ((rx_id & 0x3) == 0) { + rte_ngbe_prefetch(&rx_ring[rx_id]); + rte_ngbe_prefetch(&sw_ring[rx_id]); + } + + rxm = rxe->mbuf; + rxe->mbuf = nmb; + dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); + NGBE_RXD_HDRADDR(rxdp, 0); + NGBE_RXD_PKTADDR(rxdp, dma_addr); + + /* + * Initialize the returned mbuf. + * 1) setup generic mbuf fields: + * - number of segments, + * - next segment, + * - packet length, + * - RX port identifier. + * 2) integrate hardware offload data, if any: + * - RSS flag & hash, + * - IP checksum flag, + * - VLAN TCI, if any, + * - error flags. + */ + pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) - + rxq->crc_len); + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); + rxm->nb_segs = 1; + rxm->next = NULL; + rxm->pkt_len = pkt_len; + rxm->data_len = pkt_len; + rxm->port = rxq->port_id; + + pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0); + /* Only valid if PKT_RX_VLAN set in pkt_flags */ + rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag); + + pkt_flags = rx_desc_status_to_pkt_flags(staterr, + rxq->vlan_flags); + pkt_flags |= rx_desc_error_to_pkt_flags(staterr); + pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); + rxm->ol_flags = pkt_flags; + rxm->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, + rxq->pkt_type_mask); + + if (likely(pkt_flags & PKT_RX_RSS_HASH)) + rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1); + + /* + * Store the mbuf address into the next entry of the array + * of returned packets. + */ + rx_pkts[nb_rx++] = rxm; + } + rxq->rx_tail = rx_id; + + /* + * If the number of free RX descriptors is greater than the RX free + * threshold of the queue, advance the Receive Descriptor Tail (RDT) + * register. + * Update the RDT with the value of the last processed RX descriptor + * minus 1, to guarantee that the RDT register is never equal to the + * RDH register, which creates a "full" ring situation from the + * hardware point of view... + */ + nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); + if (nb_hold > rxq->rx_free_thresh) { + PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " + "nb_hold=%u nb_rx=%u", + (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, + (uint16_t)rx_id, (uint16_t)nb_hold, + (uint16_t)nb_rx); + rx_id = (uint16_t)((rx_id == 0) ? + (rxq->nb_rx_desc - 1) : (rx_id - 1)); + ngbe_set32(rxq->rdt_reg_addr, rx_id); + nb_hold = 0; + } + rxq->nb_rx_hold = nb_hold; + return nb_rx; +} + /********************************************************************* * * Queue management functions diff --git a/drivers/net/ngbe/ngbe_rxtx.h b/drivers/net/ngbe/ngbe_rxtx.h index f30da10ae3..d6b9127cb4 100644 --- a/drivers/net/ngbe/ngbe_rxtx.h +++ b/drivers/net/ngbe/ngbe_rxtx.h @@ -43,6 +43,85 @@ struct ngbe_rx_desc { } qw1; /* also as r.hdr_addr */ }; +/* @ngbe_rx_desc.qw0 */ +#define NGBE_RXD_PKTADDR(rxd, v) \ + (((volatile __le64 *)(rxd))[0] = cpu_to_le64(v)) + +/* @ngbe_rx_desc.qw1 */ +#define NGBE_RXD_HDRADDR(rxd, v) \ + (((volatile __le64 *)(rxd))[1] = cpu_to_le64(v)) + +/* @ngbe_rx_desc.dw0 */ +#define NGBE_RXD_RSSTYPE(dw) RS(dw, 0, 0xF) +#define NGBE_RSSTYPE_NONE 0 +#define NGBE_RSSTYPE_IPV4TCP 1 +#define NGBE_RSSTYPE_IPV4 2 +#define NGBE_RSSTYPE_IPV6TCP 3 +#define NGBE_RSSTYPE_IPV4SCTP 4 +#define NGBE_RSSTYPE_IPV6 5 +#define NGBE_RSSTYPE_IPV6SCTP 6 +#define NGBE_RSSTYPE_IPV4UDP 7 +#define NGBE_RSSTYPE_IPV6UDP 8 +#define NGBE_RSSTYPE_FDIR 15 +#define NGBE_RXD_SECTYPE(dw) RS(dw, 4, 0x3) +#define NGBE_RXD_SECTYPE_NONE LS(0, 4, 0x3) +#define NGBE_RXD_SECTYPE_IPSECESP LS(2, 4, 0x3) +#define NGBE_RXD_SECTYPE_IPSECAH LS(3, 4, 0x3) +#define NGBE_RXD_TPIDSEL(dw) RS(dw, 6, 0x7) +#define NGBE_RXD_PTID(dw) RS(dw, 9, 0xFF) +#define NGBE_RXD_RSCCNT(dw) RS(dw, 17, 0xF) +#define NGBE_RXD_HDRLEN(dw) RS(dw, 21, 0x3FF) +#define NGBE_RXD_SPH MS(31, 0x1) + +/* @ngbe_rx_desc.dw1 */ +/** bit 0-31, as rss hash when **/ +#define NGBE_RXD_RSSHASH(rxd) ((rxd)->qw0.dw1) + +/** bit 0-31, as ip csum when **/ +#define NGBE_RXD_IPID(rxd) ((rxd)->qw0.hi.ipid) +#define NGBE_RXD_CSUM(rxd) ((rxd)->qw0.hi.csum) + +/* @ngbe_rx_desc.dw2 */ +#define NGBE_RXD_STATUS(rxd) ((rxd)->qw1.lo.status) +/** bit 0-1 **/ +#define NGBE_RXD_STAT_DD MS(0, 0x1) /* Descriptor Done */ +#define NGBE_RXD_STAT_EOP MS(1, 0x1) /* End of Packet */ +/** bit 2-31, when EOP=0 **/ +#define NGBE_RXD_NEXTP_RESV(v) LS(v, 2, 0x3) +#define NGBE_RXD_NEXTP(dw) RS(dw, 4, 0xFFFF) /* Next Descriptor */ +/** bit 2-31, when EOP=1 **/ +#define NGBE_RXD_PKT_CLS_MASK MS(2, 0x7) /* Packet Class */ +#define NGBE_RXD_PKT_CLS_TC_RSS LS(0, 2, 0x7) /* RSS Hash */ +#define NGBE_RXD_PKT_CLS_FLM LS(1, 2, 0x7) /* FDir Match */ +#define NGBE_RXD_PKT_CLS_SYN LS(2, 2, 0x7) /* TCP Sync */ +#define NGBE_RXD_PKT_CLS_5TUPLE LS(3, 2, 0x7) /* 5 Tuple */ +#define NGBE_RXD_PKT_CLS_ETF LS(4, 2, 0x7) /* Ethertype Filter */ +#define NGBE_RXD_STAT_VLAN MS(5, 0x1) /* IEEE VLAN Packet */ +#define NGBE_RXD_STAT_UDPCS MS(6, 0x1) /* UDP xsum calculated */ +#define NGBE_RXD_STAT_L4CS MS(7, 0x1) /* L4 xsum calculated */ +#define NGBE_RXD_STAT_IPCS MS(8, 0x1) /* IP xsum calculated */ +#define NGBE_RXD_STAT_PIF MS(9, 0x1) /* Non-unicast address */ +#define NGBE_RXD_STAT_EIPCS MS(10, 0x1) /* Encap IP xsum calculated */ +#define NGBE_RXD_STAT_VEXT MS(11, 0x1) /* Multi-VLAN */ +#define NGBE_RXD_STAT_IPV6EX MS(12, 0x1) /* IPv6 with option header */ +#define NGBE_RXD_STAT_LLINT MS(13, 0x1) /* Pkt caused LLI */ +#define NGBE_RXD_STAT_1588 MS(14, 0x1) /* IEEE1588 Time Stamp */ +#define NGBE_RXD_STAT_SECP MS(15, 0x1) /* Security Processing */ +#define NGBE_RXD_STAT_LB MS(16, 0x1) /* Loopback Status */ +/*** bit 17-30, when PTYPE=IP ***/ +#define NGBE_RXD_STAT_BMC MS(17, 0x1) /* PTYPE=IP, BMC status */ +#define NGBE_RXD_ERR_HBO MS(23, 0x1) /* Header Buffer Overflow */ +#define NGBE_RXD_ERR_EIPCS MS(26, 0x1) /* Encap IP header error */ +#define NGBE_RXD_ERR_SECERR MS(27, 0x1) /* macsec or ipsec error */ +#define NGBE_RXD_ERR_RXE MS(29, 0x1) /* Any MAC Error */ +#define NGBE_RXD_ERR_L4CS MS(30, 0x1) /* TCP/UDP xsum error */ +#define NGBE_RXD_ERR_IPCS MS(31, 0x1) /* IP xsum error */ +#define NGBE_RXD_ERR_CSUM(dw) RS(dw, 30, 0x3) + +/* @ngbe_rx_desc.dw3 */ +#define NGBE_RXD_LENGTH(rxd) ((rxd)->qw1.hi.len) +#define NGBE_RXD_VLAN(rxd) ((rxd)->qw1.hi.tag) + /***************************************************************************** * Transmit Descriptor *****************************************************************************/ @@ -68,11 +147,40 @@ struct ngbe_tx_desc { __le32 dw3; /* r.olinfo_status, w.status */ }; +/* @ngbe_tx_desc.dw2 */ +#define NGBE_TXD_DATLEN(v) ((0xFFFF & (v))) /* data buffer length */ +#define NGBE_TXD_1588 ((0x1) << 19) /* IEEE1588 time stamp */ +#define NGBE_TXD_DATA ((0x0) << 20) /* data descriptor */ +#define NGBE_TXD_EOP ((0x1) << 24) /* End of Packet */ +#define NGBE_TXD_FCS ((0x1) << 25) /* Insert FCS */ +#define NGBE_TXD_LINKSEC ((0x1) << 26) /* Insert LinkSec */ +#define NGBE_TXD_ECU ((0x1) << 28) /* forward to ECU */ +#define NGBE_TXD_CNTAG ((0x1) << 29) /* insert CN tag */ +#define NGBE_TXD_VLE ((0x1) << 30) /* insert VLAN tag */ +#define NGBE_TXD_TSE ((0x1) << 31) /* transmit segmentation */ + +#define NGBE_TXD_FLAGS (NGBE_TXD_FCS | NGBE_TXD_EOP) + +/* @ngbe_tx_desc.dw3 */ +#define NGBE_TXD_DD_UNUSED NGBE_TXD_DD +#define NGBE_TXD_IDX_UNUSED(v) NGBE_TXD_IDX(v) +#define NGBE_TXD_CC ((0x1) << 7) /* check context */ +#define NGBE_TXD_IPSEC ((0x1) << 8) /* request ipsec offload */ +#define NGBE_TXD_L4CS ((0x1) << 9) /* insert TCP/UDP/SCTP csum */ +#define NGBE_TXD_IPCS ((0x1) << 10) /* insert IPv4 csum */ +#define NGBE_TXD_EIPCS ((0x1) << 11) /* insert outer IP csum */ +#define NGBE_TXD_MNGFLT ((0x1) << 12) /* enable management filter */ +#define NGBE_TXD_PAYLEN(v) ((0x7FFFF & (v)) << 13) /* payload length */ + +#define RTE_PMD_NGBE_TX_MAX_BURST 32 #define RTE_PMD_NGBE_RX_MAX_BURST 32 +#define RTE_NGBE_TX_MAX_FREE_BUF_SZ 64 #define RX_RING_SZ ((NGBE_RING_DESC_MAX + RTE_PMD_NGBE_RX_MAX_BURST) * \ sizeof(struct ngbe_rx_desc)) +#define rte_packet_prefetch(p) rte_prefetch1(p) + #define NGBE_TX_MAX_SEG 40 /** @@ -124,6 +232,8 @@ struct ngbe_rx_queue { uint8_t crc_len; /**< 0 if CRC stripped, 4 otherwise. */ uint8_t drop_en; /**< If not 0, set SRRCTL.Drop_En. */ uint8_t rx_deferred_start; /**< not in global dev start. */ + /** flags to set in mbuf when a vlan is detected. */ + uint64_t vlan_flags; uint64_t offloads; /**< Rx offloads with DEV_RX_OFFLOAD_* */ /** need to alloc dummy mbuf, for wraparound when scanning hw ring */ struct rte_mbuf fake_mbuf; -- 2.27.0