From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A4B45465C0; Fri, 18 Apr 2025 11:09:27 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 7BFEE406B4; Fri, 18 Apr 2025 11:07:45 +0200 (CEST) Received: from mail-m16.vip.163.com (mail-m16.vip.163.com [1.95.21.4]) by mails.dpdk.org (Postfix) with ESMTP id 9D3EB409FA for ; Fri, 18 Apr 2025 11:07:40 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=vip.163.com; s=s110527; h=From:Subject:Date:Message-ID: MIME-Version; bh=7DD5UippSAdg1aWB5kp7EmbmFjv302tndyd7UmOHVis=; b=Em5w+yJwjWnABL8XexZ3r105jP2iJXMC7FRY+XusWQzhYcq5Djl8hh6xy1pTt1 usFfXWi4bJZE/D57UJF+bu164LU0za+JK9emjF4XC8qrVaYE+VyZeM97XAP86zao 07R8i9+53KH/Gy9+hss1qaZaKPBHPdRI84iOEX/6mLIDE= Received: from localhost.localdomain (unknown [114.116.198.59]) by gzsmtp2 (Coremail) with SMTP id As8vCgDXqJVFFgJorqzGAg--.15042S18; Fri, 18 Apr 2025 17:07:39 +0800 (CST) From: Feifei Wang To: dev@dpdk.org Cc: Feifei Wang , Yi Chen , Xin Wang Subject: [RFC 14/18] net/hinic3: add Rx/Tx functions Date: Fri, 18 Apr 2025 17:06:00 +0800 Message-ID: <20250418090621.9638-15-wff_light@vip.163.com> X-Mailer: git-send-email 2.47.0.windows.2 In-Reply-To: <20250418090621.9638-1-wff_light@vip.163.com> References: <20250418090621.9638-1-wff_light@vip.163.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-CM-TRANSID: As8vCgDXqJVFFgJorqzGAg--.15042S18 X-Coremail-Antispam: 1Uf129KBjvAXoWfCryfXFy8tw43XF1Utr1rtFb_yoW5Zr4fGo WfXr13tr1SvryxCrWj9w4kuFn2qws0vFW5JwnYyF42vF1UAryYga9xGw1Fqa48XrWFk3Wx Aa45JwnFy39xJ3y5n29KB7ZKAUJUUUU8529EdanIXcx71UUUUU7v73VFW2AGmfu7bjvjm3 AaLaJ3UbIYCTnIWIevJa73UjIFyTuYvjxUISoGDUUUU X-Originating-IP: [114.116.198.59] X-CM-SenderInfo: pziiszhljk3qxylshiywtou0bp/1tbiHwozCmgB7Bq-qQAAsl X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Feifei Wang This patch add package sending and receiving function codes. Signed-off-by: Feifei Wang Signed-off-by: Yi Chen Reviewed-by: Xin Wang --- drivers/net/hinic3/hinic3_ethdev.c | 9 +- drivers/net/hinic3/hinic3_rx.c | 301 +++++++++++- drivers/net/hinic3/hinic3_tx.c | 754 +++++++++++++++++++++++++++++ drivers/net/hinic3/hinic3_tx.h | 1 + 4 files changed, 1054 insertions(+), 11 deletions(-) diff --git a/drivers/net/hinic3/hinic3_ethdev.c b/drivers/net/hinic3/hinic3_ethdev.c index de380dddbb..7cd101e5c3 100644 --- a/drivers/net/hinic3/hinic3_ethdev.c +++ b/drivers/net/hinic3/hinic3_ethdev.c @@ -21,9 +21,9 @@ #include "base/hinic3_hw_comm.h" #include "base/hinic3_nic_cfg.h" #include "base/hinic3_nic_event.h" -#include "hinic3_pmd_nic_io.h" -#include "hinic3_pmd_tx.h" -#include "hinic3_pmd_rx.h" +#include "hinic3_nic_io.h" +#include "hinic3_tx.h" +#include "hinic3_rx.h" #include "hinic3_ethdev.h" #define HINIC3_MIN_RX_BUF_SIZE 1024 @@ -3337,6 +3337,9 @@ hinic3_dev_init(struct rte_eth_dev *eth_dev) PMD_DRV_LOG(INFO, "Network Interface pmd driver version: %s", HINIC3_PMD_DRV_VERSION); + eth_dev->rx_pkt_burst = hinic3_recv_pkts; + eth_dev->tx_pkt_burst = hinic3_xmit_pkts; + return hinic3_func_init(eth_dev); } diff --git a/drivers/net/hinic3/hinic3_rx.c b/drivers/net/hinic3/hinic3_rx.c index a1dc960236..318d9aadc3 100644 --- a/drivers/net/hinic3/hinic3_rx.c +++ b/drivers/net/hinic3/hinic3_rx.c @@ -5,14 +5,14 @@ #include #include "base/hinic3_compat.h" -#include "base/hinic3_pmd_hwif.h" -#include "base/hinic3_pmd_hwdev.h" -#include "base/hinic3_pmd_wq.h" -#include "base/hinic3_pmd_nic_cfg.h" -#include "hinic3_pmd_nic_io.h" -#include "hinic3_pmd_ethdev.h" -#include "hinic3_pmd_tx.h" -#include "hinic3_pmd_rx.h" +#include "base/hinic3_hwif.h" +#include "base/hinic3_hwdev.h" +#include "base/hinic3_wq.h" +#include "base/hinic3_nic_cfg.h" +#include "hinic3_nic_io.h" +#include "hinic3_ethdev.h" +#include "hinic3_tx.h" +#include "hinic3_rx.h" /** * Get wqe from receive queue. @@ -809,3 +809,288 @@ hinic3_start_rq(struct rte_eth_dev *eth_dev, struct hinic3_rxq *rxq) return err; } + + +static inline u64 +hinic3_rx_vlan(u32 offload_type, u32 vlan_len, u16 *vlan_tci) +{ + uint16_t vlan_tag; + + vlan_tag = HINIC3_GET_RX_VLAN_TAG(vlan_len); + if (!HINIC3_GET_RX_VLAN_OFFLOAD_EN(offload_type) || vlan_tag == 0) { + *vlan_tci = 0; + return 0; + } + + *vlan_tci = vlan_tag; + + return HINIC3_PKT_RX_VLAN | HINIC3_PKT_RX_VLAN_STRIPPED; +} + +static inline u64 +hinic3_rx_csum(uint32_t status, struct hinic3_rxq *rxq) +{ + struct hinic3_nic_dev *nic_dev = rxq->nic_dev; + u32 csum_err; + u64 flags; + + if (unlikely(!(nic_dev->rx_csum_en & HINIC3_DEFAULT_RX_CSUM_OFFLOAD))) + return HINIC3_PKT_RX_IP_CKSUM_UNKNOWN; + + csum_err = HINIC3_GET_RX_CSUM_ERR(status); + if (likely(csum_err == 0)) + return (HINIC3_PKT_RX_IP_CKSUM_GOOD | + HINIC3_PKT_RX_L4_CKSUM_GOOD); + + /* + * If bypass bit is set, all other err status indications should be + * ignored. + */ + if (unlikely(csum_err & HINIC3_RX_CSUM_HW_CHECK_NONE)) + return HINIC3_PKT_RX_IP_CKSUM_UNKNOWN; + + flags = 0; + + /* IP checksum error. */ + if (csum_err & HINIC3_RX_CSUM_IP_CSUM_ERR) { + flags |= HINIC3_PKT_RX_IP_CKSUM_BAD; + rxq->rxq_stats.csum_errors++; + } + + /* L4 checksum error. */ + if ((csum_err & HINIC3_RX_CSUM_TCP_CSUM_ERR) || + (csum_err & HINIC3_RX_CSUM_UDP_CSUM_ERR) || + (csum_err & HINIC3_RX_CSUM_SCTP_CRC_ERR)) { + flags |= HINIC3_PKT_RX_L4_CKSUM_BAD; + rxq->rxq_stats.csum_errors++; + } + + if (unlikely(csum_err == HINIC3_RX_CSUM_IPSU_OTHER_ERR)) + rxq->rxq_stats.other_errors++; + + return flags; +} + +static inline u64 +hinic3_rx_rss_hash(u32 offload_type, u32 rss_hash_value, u32 *rss_hash) +{ + u32 rss_type; + + rss_type = HINIC3_GET_RSS_TYPES(offload_type); + if (likely(rss_type != 0)) { + *rss_hash = rss_hash_value; + return HINIC3_PKT_RX_RSS_HASH; + } + + return 0; +} + +static void +hinic3_recv_jumbo_pkt(struct hinic3_rxq *rxq, struct rte_mbuf *head_mbuf, + u32 remain_pkt_len) +{ + struct rte_mbuf *cur_mbuf = NULL; + struct rte_mbuf *rxm = NULL; + struct hinic3_rx_info *rx_info = NULL; + u16 sw_ci, rx_buf_len = rxq->buf_len; + u32 pkt_len; + + while (remain_pkt_len > 0) { + sw_ci = hinic3_get_rq_local_ci(rxq); + rx_info = &rxq->rx_info[sw_ci]; + + hinic3_update_rq_local_ci(rxq, 1); + + pkt_len = remain_pkt_len > rx_buf_len ? rx_buf_len + : remain_pkt_len; + remain_pkt_len -= pkt_len; + + cur_mbuf = rx_info->mbuf; + cur_mbuf->data_len = (u16)pkt_len; + cur_mbuf->next = NULL; + + head_mbuf->pkt_len += cur_mbuf->data_len; + head_mbuf->nb_segs++; +#ifdef HINIC3_XSTAT_MBUF_USE + rxq->rxq_stats.rx_free_mbuf_bytes++; +#endif + if (!rxm) + head_mbuf->next = cur_mbuf; + else + rxm->next = cur_mbuf; + + rxm = cur_mbuf; + } +} + +int +hinic3_start_all_rqs(struct rte_eth_dev *eth_dev) +{ + struct hinic3_nic_dev *nic_dev = NULL; + struct hinic3_rxq *rxq = NULL; + int err = 0; + int i; + + nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev); + + for (i = 0; i < nic_dev->num_rqs; i++) { + rxq = eth_dev->data->rx_queues[i]; + hinic3_add_rq_to_rx_queue_list(nic_dev, rxq->q_id); + err = hinic3_rearm_rxq_mbuf(rxq); + if (err) { + PMD_DRV_LOG(ERR, + "Fail to alloc mbuf for Rx queue %d, " + "qid = %u, need_mbuf: %d", + i, rxq->q_id, rxq->q_depth); + goto out; + } + hinic3_dev_rx_queue_intr_enable(eth_dev, rxq->q_id); + eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + if (nic_dev->rss_state == HINIC3_RSS_ENABLE) { + err = hinic3_refill_indir_rqid(rxq); + if (err) { + PMD_DRV_LOG(ERR, + "Refill rq to indrect table failed, " + "eth_dev:%s, queue_idx:%d, err:%d", + rxq->nic_dev->dev_name, rxq->q_id, err); + goto out; + } + } + + return 0; +out: + for (i = 0; i < nic_dev->num_rqs; i++) { + rxq = eth_dev->data->rx_queues[i]; + hinic3_remove_rq_from_rx_queue_list(nic_dev, rxq->q_id); + hinic3_free_rxq_mbufs(rxq); + hinic3_dev_rx_queue_intr_disable(eth_dev, rxq->q_id); + eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + return err; +} + +#define HINIC3_RX_EMPTY_THRESHOLD 3 +u16 +hinic3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts) +{ + struct hinic3_rxq *rxq = rx_queue; + struct hinic3_rx_info *rx_info = NULL; + volatile struct hinic3_rq_cqe *rx_cqe = NULL; + struct rte_mbuf *rxm = NULL; + u16 sw_ci, rx_buf_len, wqebb_cnt = 0, pkts = 0; + u32 status, pkt_len, vlan_len, offload_type, lro_num; + u64 rx_bytes = 0; + u32 hash_value; + +#ifdef HINIC3_XSTAT_PROF_RX + uint64_t t1 = rte_get_tsc_cycles(); + uint64_t t2; +#endif + if (((rte_get_timer_cycles() - rxq->rxq_stats.tsc) < rxq->wait_time_cycle) && + rxq->rxq_stats.empty >= HINIC3_RX_EMPTY_THRESHOLD) + goto out; + + sw_ci = hinic3_get_rq_local_ci(rxq); + rx_buf_len = rxq->buf_len; + + while (pkts < nb_pkts) { + rx_cqe = &rxq->rx_cqe[sw_ci]; + status = hinic3_hw_cpu32((u32)(rte_atomic_load_explicit(&rx_cqe->status, + rte_memory_order_acquire))); + if (!HINIC3_GET_RX_DONE(status)) { + rxq->rxq_stats.empty++; + break; + } + + vlan_len = hinic3_hw_cpu32(rx_cqe->vlan_len); + + pkt_len = HINIC3_GET_RX_PKT_LEN(vlan_len); + + rx_info = &rxq->rx_info[sw_ci]; + rxm = rx_info->mbuf; + + /* 1. Next ci point and prefetch. */ + sw_ci++; + sw_ci &= rxq->q_mask; + + /* 2. Prefetch next mbuf first 64B. */ + rte_prefetch0(rxq->rx_info[sw_ci].mbuf); + + /* 3. Jumbo frame process. */ + if (likely(pkt_len <= (u32)rx_buf_len)) { + rxm->data_len = (u16)pkt_len; + rxm->pkt_len = pkt_len; + wqebb_cnt++; + } else { + rxm->data_len = rx_buf_len; + rxm->pkt_len = rx_buf_len; + + /* + * If receive jumbo, updating ci will be done by + * hinic3_recv_jumbo_pkt function. + */ + hinic3_update_rq_local_ci(rxq, wqebb_cnt + 1); + wqebb_cnt = 0; + hinic3_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len); + sw_ci = hinic3_get_rq_local_ci(rxq); + } + + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rxm->port = rxq->port_id; + + /* 4. Rx checksum offload. */ + rxm->ol_flags |= hinic3_rx_csum(status, rxq); + + /* 5. Vlan offload. */ + offload_type = hinic3_hw_cpu32(rx_cqe->offload_type); + + rxm->ol_flags |= + hinic3_rx_vlan(offload_type, vlan_len, &rxm->vlan_tci); + + /* 6. RSS. */ + hash_value = hinic3_hw_cpu32(rx_cqe->hash_val); + rxm->ol_flags |= hinic3_rx_rss_hash(offload_type, hash_value, + &rxm->hash.rss); + /* 8. LRO. */ + lro_num = HINIC3_GET_RX_NUM_LRO(status); + if (unlikely(lro_num != 0)) { + rxm->ol_flags |= HINIC3_PKT_RX_LRO; + rxm->tso_segsz = pkt_len / lro_num; + } + + rx_cqe->status = 0; + + rx_bytes += pkt_len; + rx_pkts[pkts++] = rxm; + } + + if (pkts) { + /* 9. Update local ci. */ + hinic3_update_rq_local_ci(rxq, wqebb_cnt); + + /* Update packet stats. */ + rxq->rxq_stats.packets += pkts; + rxq->rxq_stats.bytes += rx_bytes; + rxq->rxq_stats.empty = 0; +#ifdef HINIC3_XSTAT_MBUF_USE + rxq->rxq_stats.rx_free_mbuf_bytes += pkts; +#endif + } + rxq->rxq_stats.burst_pkts = pkts; + rxq->rxq_stats.tsc = rte_get_timer_cycles(); +out: + /* 10. Rearm mbuf to rxq. */ + hinic3_rearm_rxq_mbuf(rxq); + +#ifdef HINIC3_XSTAT_PROF_RX + /* Do profiling stats. */ + t2 = rte_get_tsc_cycles(); + rxq->rxq_stats.app_tsc = t1 - rxq->prof_rx_end_tsc; + rxq->prof_rx_end_tsc = t2; + rxq->rxq_stats.pmd_tsc = t2 - t1; +#endif + + return pkts; +} diff --git a/drivers/net/hinic3/hinic3_tx.c b/drivers/net/hinic3/hinic3_tx.c index 6f8c42e0c3..c2157ab4b9 100644 --- a/drivers/net/hinic3/hinic3_tx.c +++ b/drivers/net/hinic3/hinic3_tx.c @@ -60,6 +60,98 @@ hinic3_get_sq_hw_ci(struct hinic3_txq *sq) return MASKED_QUEUE_IDX(sq, hinic3_hw_cpu16(*sq->ci_vaddr_base)); } +static void * +hinic3_get_sq_wqe(struct hinic3_txq *sq, struct hinic3_wqe_info *wqe_info) +{ + u16 cur_pi = MASKED_QUEUE_IDX(sq, sq->prod_idx); + u32 end_pi; + + end_pi = cur_pi + wqe_info->wqebb_cnt; + sq->prod_idx += wqe_info->wqebb_cnt; + + wqe_info->owner = (u8)(sq->owner); + wqe_info->pi = cur_pi; + wqe_info->wrapped = 0; + + if (unlikely(end_pi >= sq->q_depth)) { + sq->owner = !sq->owner; + + if (likely(end_pi > sq->q_depth)) + wqe_info->wrapped = (u8)(sq->q_depth - cur_pi); + } + + return NIC_WQE_ADDR(sq, cur_pi); +} + +static inline void +hinic3_put_sq_wqe(struct hinic3_txq *sq, struct hinic3_wqe_info *wqe_info) +{ + if (wqe_info->owner != sq->owner) + sq->owner = wqe_info->owner; + + sq->prod_idx -= wqe_info->wqebb_cnt; +} + +/** + * Sets the WQE combination information in the transmit queue (SQ). + * + * @param[in] txq + * Point to send queue. + * @param[out] wqe_combo + * Point to wqe_combo of send queue(SQ). + * @param[in] wqe + * Point to wqe of send queue(SQ). + * @param[in] wqe_info + * Point to wqe_info of send queue(SQ). + */ +static void +hinic3_set_wqe_combo(struct hinic3_txq *txq, + struct hinic3_sq_wqe_combo *wqe_combo, + struct hinic3_sq_wqe *wqe, + struct hinic3_wqe_info *wqe_info) +{ + wqe_combo->hdr = &wqe->compact_wqe.wqe_desc; + + if (wqe_info->offload) { + if (wqe_info->wrapped == HINIC3_TX_TASK_WRAPPED) { + wqe_combo->task = (struct hinic3_sq_task *) + (void *)txq->sq_head_addr; + wqe_combo->bds_head = (struct hinic3_sq_bufdesc *) + (void *)(txq->sq_head_addr + txq->wqebb_size); + } else if (wqe_info->wrapped == HINIC3_TX_BD_DESC_WRAPPED) { + wqe_combo->task = &wqe->extend_wqe.task; + wqe_combo->bds_head = (struct hinic3_sq_bufdesc *) + (void *)(txq->sq_head_addr); + } else { + wqe_combo->task = &wqe->extend_wqe.task; + wqe_combo->bds_head = wqe->extend_wqe.buf_desc; + } + + wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE; + wqe_combo->task_type = SQ_WQE_TASKSECT_16BYTES; + + return; + } + + if (wqe_info->wrapped == HINIC3_TX_TASK_WRAPPED) { + wqe_combo->bds_head = (struct hinic3_sq_bufdesc *) + (void *)(txq->sq_head_addr); + } else { + wqe_combo->bds_head = + (struct hinic3_sq_bufdesc *)(&wqe->extend_wqe.task); + } + + if (wqe_info->wqebb_cnt > 1) { + wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE; + wqe_combo->task_type = SQ_WQE_TASKSECT_46BITS; + + /* This section used as vlan insert, needs to clear. */ + wqe_combo->bds_head->rsvd = 0; + } else { + wqe_combo->wqe_type = SQ_WQE_COMPACT_TYPE; + } +} + int hinic3_start_all_sqs(struct rte_eth_dev *eth_dev) { @@ -220,6 +312,668 @@ hinic3_tx_done_cleanup(void *txq, u32 free_cnt) return hinic3_xmit_mbuf_cleanup(tx_queue, try_free_cnt); } +/** + * Prepare the data packet to be sent and calculate the internal L3 offset. + * + * @param[in] mbuf + * Point to the mbuf to be processed. + * @param[out] inner_l3_offset + * Inner(IP Layer) L3 layer offset. + * @return + * 0 as success, -EINVAL as failure. + */ +static int +hinic3_tx_offload_pkt_prepare(struct rte_mbuf *mbuf, u16 *inner_l3_offset) +{ + uint64_t ol_flags = mbuf->ol_flags; + + /* Only support vxlan offload. */ + if ((ol_flags & HINIC3_PKT_TX_TUNNEL_MASK) && + (!(ol_flags & HINIC3_PKT_TX_TUNNEL_VXLAN))) + return -EINVAL; + +#ifdef RTE_LIBRTE_ETHDEV_DEBUG + if (rte_validate_tx_offload(mbuf) != 0) + return -EINVAL; +#endif + /* Support tunnel. */ + if ((ol_flags & HINIC3_PKT_TX_TUNNEL_MASK)) { + if ((ol_flags & HINIC3_PKT_TX_OUTER_IP_CKSUM) || + (ol_flags & HINIC3_PKT_TX_OUTER_IPV6) || + (ol_flags & HINIC3_PKT_TX_TCP_SEG)) { + /* + * For this senmatic, l2_len of mbuf means + * len(out_udp + vxlan + in_eth). + */ + *inner_l3_offset = mbuf->l2_len + mbuf->outer_l2_len + + mbuf->outer_l3_len; + } else { + /* + * For this senmatic, l2_len of mbuf means + * len(out_eth + out_ip + out_udp + vxlan + in_eth). + */ + *inner_l3_offset = mbuf->l2_len; + } + } else { + /* For non-tunnel type pkts. */ + *inner_l3_offset = mbuf->l2_len; + } + + return 0; +} + +static inline void +hinic3_set_vlan_tx_offload(struct hinic3_sq_task *task, u16 vlan_tag, + u8 vlan_type) +{ + task->vlan_offload = SQ_TASK_INFO3_SET(vlan_tag, VLAN_TAG) | + SQ_TASK_INFO3_SET(vlan_type, VLAN_TYPE) | + SQ_TASK_INFO3_SET(1U, VLAN_TAG_VALID); +} + +/** + * Set the corresponding offload information based on ol_flags of the mbuf. + * + * @param[in] mbuf + * Point to the mbuf for which offload needs to be set in the sending queue. + * @param[out] task + * Point to task of send queue(SQ). + * @param[out] wqe_info + * Point to wqe_info of send queue(SQ). + * @return + * 0 as success, -EINVAL as failure. + */ +static int +hinic3_set_tx_offload(struct rte_mbuf *mbuf, struct hinic3_sq_task *task, + struct hinic3_wqe_info *wqe_info) +{ + uint64_t ol_flags = mbuf->ol_flags; + u16 pld_offset = 0; + u32 queue_info = 0; + u16 vlan_tag; + + task->pkt_info0 = 0; + task->ip_identify = 0; + task->pkt_info2 = 0; + task->vlan_offload = 0; + + /* Vlan offload. */ + if (unlikely(ol_flags & HINIC3_PKT_TX_VLAN_PKT)) { + vlan_tag = mbuf->vlan_tci; + hinic3_set_vlan_tx_offload(task, vlan_tag, HINIC3_TX_TPID0); + task->vlan_offload = hinic3_hw_be32(task->vlan_offload); + } + /* Cksum offload. */ + if (!(ol_flags & HINIC3_TX_CKSUM_OFFLOAD_MASK)) + return 0; + + /* Tso offload. */ + if (ol_flags & HINIC3_PKT_TX_TCP_SEG) { + pld_offset = wqe_info->payload_offset; + if ((pld_offset >> 1) > MAX_PAYLOAD_OFFSET) + return -EINVAL; + + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L4_EN); + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L3_EN); + + queue_info |= SQ_CTRL_QUEUE_INFO_SET(1U, TSO); + queue_info |= SQ_CTRL_QUEUE_INFO_SET(pld_offset >> 1, PLDOFF); + + /* Set MSS value. */ + queue_info = SQ_CTRL_QUEUE_INFO_CLEAR(queue_info, MSS); + queue_info |= SQ_CTRL_QUEUE_INFO_SET(mbuf->tso_segsz, MSS); + } else { + if (ol_flags & HINIC3_PKT_TX_IP_CKSUM) + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L3_EN); + + switch (ol_flags & HINIC3_PKT_TX_L4_MASK) { + case HINIC3_PKT_TX_TCP_CKSUM: + case HINIC3_PKT_TX_UDP_CKSUM: + case HINIC3_PKT_TX_SCTP_CKSUM: + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L4_EN); + break; + + case HINIC3_PKT_TX_L4_NO_CKSUM: + break; + + default: + PMD_DRV_LOG(INFO, "not support pkt type"); + return -EINVAL; + } + } + + /* For vxlan, also can support PKT_TX_TUNNEL_GRE, etc. */ + switch (ol_flags & HINIC3_PKT_TX_TUNNEL_MASK) { + case HINIC3_PKT_TX_TUNNEL_VXLAN: + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, TUNNEL_FLAG); + break; + + case 0: + break; + + default: + /* For non UDP/GRE tunneling, drop the tunnel packet. */ + PMD_DRV_LOG(INFO, "not support tunnel pkt type"); + return -EINVAL; + } + + if (ol_flags & HINIC3_PKT_TX_OUTER_IP_CKSUM) + task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, OUT_L3_EN); + + task->pkt_info0 = hinic3_hw_be32(task->pkt_info0); + task->pkt_info2 = hinic3_hw_be32(task->pkt_info2); + wqe_info->queue_info = queue_info; + + return 0; +} + +/** + * Check whether the number of segments in the mbuf is valid. + * + * @param[in] mbuf + * Point to the mbuf to be verified. + * @param[in] wqe_info + * Point to wqe_info of send queue(SQ). + * @return + * true as valid, false as invalid. + */ +static bool +hinic3_is_tso_sge_valid(struct rte_mbuf *mbuf, struct hinic3_wqe_info *wqe_info) +{ + u32 total_len, limit_len, checked_len, left_len, adjust_mss; + u32 i, max_sges, left_sges, first_len; + struct rte_mbuf *mbuf_head, *mbuf_first; + struct rte_mbuf *mbuf_pre = mbuf; + + left_sges = mbuf->nb_segs; + mbuf_head = mbuf; + mbuf_first = mbuf; + + /* Tso sge number validation. */ + if (unlikely(left_sges >= HINIC3_NONTSO_PKT_MAX_SGE)) { + checked_len = 0; + total_len = 0; + first_len = 0; + adjust_mss = mbuf->tso_segsz >= TX_MSS_MIN ? mbuf->tso_segsz + : TX_MSS_MIN; + max_sges = HINIC3_NONTSO_PKT_MAX_SGE - 1; + limit_len = adjust_mss + wqe_info->payload_offset; + + for (i = 0; (i < max_sges) && (total_len < limit_len); i++) { + total_len += mbuf->data_len; + mbuf_pre = mbuf; + mbuf = mbuf->next; + } + + /* Each continues 38 mbufs segmust do one check. */ + while (left_sges >= HINIC3_NONTSO_PKT_MAX_SGE) { + if (total_len >= limit_len) { + /* Update the limit len. */ + limit_len = adjust_mss; + /* Update checked len. */ + checked_len += first_len; + /* Record the first len. */ + first_len = mbuf_first->data_len; + /* First mbuf move to the next. */ + mbuf_first = mbuf_first->next; + /* Update total len. */ + total_len -= first_len; + left_sges--; + i--; + for (; + (i < max_sges) && (total_len < limit_len); + i++) { + total_len += mbuf->data_len; + mbuf_pre = mbuf; + mbuf = mbuf->next; + } + } else { + /* Try to copy if not valid. */ + checked_len += (total_len - mbuf_pre->data_len); + + left_len = mbuf_head->pkt_len - checked_len; + if (left_len > HINIC3_COPY_MBUF_SIZE) + return false; + wqe_info->sge_cnt = (u16)(mbuf_head->nb_segs + + i - left_sges); + wqe_info->cpy_mbuf_cnt = 1; + + return true; + } + } /**< End of while. */ + } + + wqe_info->sge_cnt = mbuf_head->nb_segs; + + return true; +} + +/** + * Checks and processes transport offload information for data packets. + * + * @param[in] mbuf + * Point to the mbuf to send. + * @param[in] wqe_info + * Point to wqe_info of send queue(SQ). + * @return + * 0 as success, -EINVAL as failure. + */ +static int +hinic3_get_tx_offload(struct rte_mbuf *mbuf, struct hinic3_wqe_info *wqe_info) +{ + uint64_t ol_flags = mbuf->ol_flags; + u16 i, total_len, inner_l3_offset = 0; + int err; + struct rte_mbuf *mbuf_pkt = NULL; + + wqe_info->sge_cnt = mbuf->nb_segs; + /* Check if the packet set available offload flags. */ + if (!(ol_flags & HINIC3_TX_OFFLOAD_MASK)) { + wqe_info->offload = 0; + return 0; + } + + wqe_info->offload = 1; + err = hinic3_tx_offload_pkt_prepare(mbuf, &inner_l3_offset); + if (err) + return err; + + /* Non tso mbuf only check sge num. */ + if (likely(!(mbuf->ol_flags & HINIC3_PKT_TX_TCP_SEG))) { + if (unlikely(mbuf->pkt_len > MAX_SINGLE_SGE_SIZE)) + /* Non tso packet len must less than 64KB. */ + return -EINVAL; + + if (likely(HINIC3_NONTSO_SEG_NUM_VALID(mbuf->nb_segs))) + /* Valid non-tso mbuf. */ + return 0; + + /* + * The number of non-tso packet fragments must be less than 38, + * and mbuf segs greater than 38 must be copied to other + * buffers. + */ + total_len = 0; + mbuf_pkt = mbuf; + for (i = 0; i < (HINIC3_NONTSO_PKT_MAX_SGE - 1); i++) { + total_len += mbuf_pkt->data_len; + mbuf_pkt = mbuf_pkt->next; + } + + /* Default support copy total 4k mbuf segs. */ + if ((u32)(total_len + (u16)HINIC3_COPY_MBUF_SIZE) < + mbuf->pkt_len) + return -EINVAL; + + wqe_info->sge_cnt = HINIC3_NONTSO_PKT_MAX_SGE; + wqe_info->cpy_mbuf_cnt = 1; + + return 0; + } + + /* Tso mbuf. */ + wqe_info->payload_offset = + inner_l3_offset + mbuf->l3_len + mbuf->l4_len; + + /* Too many mbuf segs. */ + if (unlikely(HINIC3_TSO_SEG_NUM_INVALID(mbuf->nb_segs))) + return -EINVAL; + + /* Check whether can cover all tso mbuf segs or not. */ + if (unlikely(!hinic3_is_tso_sge_valid(mbuf, wqe_info))) + return -EINVAL; + + return 0; +} + +static inline void +hinic3_set_buf_desc(struct hinic3_sq_bufdesc *buf_descs, rte_iova_t addr, + u32 len) +{ + buf_descs->hi_addr = hinic3_hw_be32(upper_32_bits(addr)); + buf_descs->lo_addr = hinic3_hw_be32(lower_32_bits(addr)); + buf_descs->len = hinic3_hw_be32(len); +} + +static inline struct rte_mbuf * +hinic3_alloc_cpy_mbuf(struct hinic3_nic_dev *nic_dev) +{ + return rte_pktmbuf_alloc(nic_dev->cpy_mpool); +} + +/** + * Copy packets in the send queue(SQ). + * + * @param[in] nic_dev + * Point to nic device. + * @param[in] mbuf + * Point to the source mbuf. + * @param[in] seg_cnt + * Number of mbuf segments to be copied. + * @result + * The address of the copied mbuf. + */ +static void * +hinic3_copy_tx_mbuf(struct hinic3_nic_dev *nic_dev, struct rte_mbuf *mbuf, + u16 sge_cnt) +{ + struct rte_mbuf *dst_mbuf; + u32 offset = 0; + u16 i; + + if (unlikely(!nic_dev->cpy_mpool)) + return NULL; + + dst_mbuf = hinic3_alloc_cpy_mbuf(nic_dev); + if (unlikely(!dst_mbuf)) + return NULL; + + dst_mbuf->data_off = 0; + dst_mbuf->data_len = 0; + for (i = 0; i < sge_cnt; i++) { + rte_memcpy((u8 *)dst_mbuf->buf_addr + offset, + (u8 *)mbuf->buf_addr + mbuf->data_off, + mbuf->data_len); + dst_mbuf->data_len += mbuf->data_len; + offset += mbuf->data_len; + mbuf = mbuf->next; + } + dst_mbuf->pkt_len = dst_mbuf->data_len; + + return dst_mbuf; +} + +/** + * Map the TX mbuf to the DMA address space and set related information for + * subsequent DMA transmission. + * + * @param[in] txq + * Point to send queue. + * @param[in] mbuf + * Point to the tx mbuf. + * @param[out] wqe_combo + * Point to send queue wqe_combo. + * @param[in] wqe_info + * Point to wqe_info of send queue(SQ). + * @result + * 0 as success, -EINVAL as failure. + */ +static int +hinic3_mbuf_dma_map_sge(struct hinic3_txq *txq, struct rte_mbuf *mbuf, + struct hinic3_sq_wqe_combo *wqe_combo, + struct hinic3_wqe_info *wqe_info) +{ + struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->hdr; + struct hinic3_sq_bufdesc *buf_desc = wqe_combo->bds_head; + + uint16_t nb_segs = wqe_info->sge_cnt - wqe_info->cpy_mbuf_cnt; + uint16_t real_segs = mbuf->nb_segs; + rte_iova_t dma_addr; + u32 i; + + for (i = 0; i < nb_segs; i++) { + if (unlikely(mbuf == NULL)) { + txq->txq_stats.mbuf_null++; + return -EINVAL; + } + + if (unlikely(mbuf->data_len == 0)) { + txq->txq_stats.sge_len0++; + return -EINVAL; + } + + dma_addr = rte_mbuf_data_iova(mbuf); + if (i == 0) { + if (wqe_combo->wqe_type == SQ_WQE_COMPACT_TYPE && + mbuf->data_len > COMPACT_WQE_MAX_CTRL_LEN) { + txq->txq_stats.sge_len_too_large++; + return -EINVAL; + } + + wqe_desc->hi_addr = + hinic3_hw_be32(upper_32_bits(dma_addr)); + wqe_desc->lo_addr = + hinic3_hw_be32(lower_32_bits(dma_addr)); + wqe_desc->ctrl_len = mbuf->data_len; + } else { + /* + * Parts of wqe is in sq bottom while parts + * of wqe is in sq head. + */ + if (unlikely(wqe_info->wrapped && + (u64)buf_desc == txq->sq_bot_sge_addr)) + buf_desc = (struct hinic3_sq_bufdesc *) + (void *)txq->sq_head_addr; + + hinic3_set_buf_desc(buf_desc, dma_addr, mbuf->data_len); + buf_desc++; + } + mbuf = mbuf->next; + } + + /* For now: support over 38 sge, copy the last 2 mbuf. */ + if (unlikely(wqe_info->cpy_mbuf_cnt != 0)) { + /* + * Copy invalid mbuf segs to a valid buffer, lost performance. + */ + txq->txq_stats.cpy_pkts += 1; + mbuf = hinic3_copy_tx_mbuf(txq->nic_dev, mbuf, + real_segs - nb_segs); + if (unlikely(!mbuf)) + return -EINVAL; + + txq->tx_info[wqe_info->pi].cpy_mbuf = mbuf; + + /* Deal with the last mbuf. */ + dma_addr = rte_mbuf_data_iova(mbuf); + if (unlikely(mbuf->data_len == 0)) { + txq->txq_stats.sge_len0++; + return -EINVAL; + } + /* + * Parts of wqe is in sq bottom while parts + * of wqe is in sq head. + */ + if (i == 0) { + wqe_desc->hi_addr = + hinic3_hw_be32(upper_32_bits(dma_addr)); + wqe_desc->lo_addr = + hinic3_hw_be32(lower_32_bits(dma_addr)); + wqe_desc->ctrl_len = mbuf->data_len; + } else { + if (unlikely(wqe_info->wrapped && + ((u64)buf_desc == txq->sq_bot_sge_addr))) + buf_desc = (struct hinic3_sq_bufdesc *) + txq->sq_head_addr; + + hinic3_set_buf_desc(buf_desc, dma_addr, mbuf->data_len); + } + } + + return 0; +} + +/** + * Sets and configures fields in the transmit queue control descriptor based on + * the WQE type. + * + * @param[out] wqe_combo + * Point to wqe_combo of send queue. + * @param[in] wqe_info + * Point to wqe_info of send queue. + */ +static void +hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo, + struct hinic3_wqe_info *wqe_info) +{ + struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->hdr; + + if (wqe_combo->wqe_type == SQ_WQE_COMPACT_TYPE) { + wqe_desc->ctrl_len |= + SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) | + SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) | + SQ_CTRL_SET(wqe_info->owner, OWNER); + wqe_desc->ctrl_len = hinic3_hw_be32(wqe_desc->ctrl_len); + + /* Compact wqe queue_info will transfer to ucode. */ + wqe_desc->queue_info = 0; + + return; + } + + wqe_desc->ctrl_len |= SQ_CTRL_SET(wqe_info->sge_cnt, BUFDESC_NUM) | + SQ_CTRL_SET(wqe_combo->task_type, TASKSECT_LEN) | + SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) | + SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) | + SQ_CTRL_SET(wqe_info->owner, OWNER); + + wqe_desc->ctrl_len = hinic3_hw_be32(wqe_desc->ctrl_len); + + wqe_desc->queue_info = wqe_info->queue_info; + wqe_desc->queue_info |= SQ_CTRL_QUEUE_INFO_SET(1U, UC); + + if (!SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS)) { + wqe_desc->queue_info |= + SQ_CTRL_QUEUE_INFO_SET(TX_MSS_DEFAULT, MSS); + } else if (SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS) < + TX_MSS_MIN) { + /* Mss should not less than 80. */ + wqe_desc->queue_info = + SQ_CTRL_QUEUE_INFO_CLEAR(wqe_desc->queue_info, MSS); + wqe_desc->queue_info |= SQ_CTRL_QUEUE_INFO_SET(TX_MSS_MIN, MSS); + } + + wqe_desc->queue_info = hinic3_hw_be32(wqe_desc->queue_info); +} + +/** + * It is responsible for sending data packets. + * + * @param[in] tx_queue + * Point to send queue. + * @param[in] tx_pkts + * Pointer to the array of data packets to be sent. + * @param[in] nb_pkts + * Number of sent packets. + * @return + * Number of actually sent packets. + */ +u16 +hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, u16 nb_pkts) +{ + struct hinic3_txq *txq = tx_queue; + struct hinic3_tx_info *tx_info = NULL; + struct rte_mbuf *mbuf_pkt = NULL; + struct hinic3_sq_wqe_combo wqe_combo = {0}; + struct hinic3_sq_wqe *sq_wqe = NULL; + struct hinic3_wqe_info wqe_info = {0}; + + u32 offload_err, free_cnt; + u64 tx_bytes = 0; + u16 free_wqebb_cnt, nb_tx; + int err; + +#ifdef HINIC3_XSTAT_PROF_TX + uint64_t t1, t2; + t1 = rte_get_tsc_cycles(); +#endif + + if (unlikely(!HINIC3_TXQ_IS_STARTED(txq))) + return 0; + + free_cnt = txq->tx_free_thresh; + /* Reclaim tx mbuf before xmit new packets. */ + if (hinic3_get_sq_free_wqebbs(txq) < txq->tx_free_thresh) + hinic3_xmit_mbuf_cleanup(txq, free_cnt); + + /* Tx loop routine. */ + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + mbuf_pkt = *tx_pkts++; + if (unlikely(hinic3_get_tx_offload(mbuf_pkt, &wqe_info))) { + txq->txq_stats.offload_errors++; + break; + } + + if (!wqe_info.offload) + wqe_info.wqebb_cnt = wqe_info.sge_cnt; + else + /* Use extended sq wqe with normal TS. */ + wqe_info.wqebb_cnt = wqe_info.sge_cnt + 1; + + free_wqebb_cnt = hinic3_get_sq_free_wqebbs(txq); + if (unlikely(wqe_info.wqebb_cnt > free_wqebb_cnt)) { + /* Reclaim again. */ + hinic3_xmit_mbuf_cleanup(txq, free_cnt); + free_wqebb_cnt = hinic3_get_sq_free_wqebbs(txq); + if (unlikely(wqe_info.wqebb_cnt > free_wqebb_cnt)) { + txq->txq_stats.tx_busy += (nb_pkts - nb_tx); + break; + } + } + + /* Get sq wqe address from wqe_page. */ + sq_wqe = hinic3_get_sq_wqe(txq, &wqe_info); + if (unlikely(!sq_wqe)) { + txq->txq_stats.tx_busy++; + break; + } + + /* Task or bd section maybe warpped for one wqe. */ + hinic3_set_wqe_combo(txq, &wqe_combo, sq_wqe, &wqe_info); + + wqe_info.queue_info = 0; + /* Fill tx packet offload into qsf and task field. */ + if (wqe_info.offload) { + offload_err = hinic3_set_tx_offload(mbuf_pkt, + wqe_combo.task, + &wqe_info); + if (unlikely(offload_err)) { + hinic3_put_sq_wqe(txq, &wqe_info); + txq->txq_stats.offload_errors++; + break; + } + } + + /* Fill sq_wqe buf_desc and bd_desc. */ + err = hinic3_mbuf_dma_map_sge(txq, mbuf_pkt, &wqe_combo, + &wqe_info); + if (err) { + hinic3_put_sq_wqe(txq, &wqe_info); + txq->txq_stats.offload_errors++; + break; + } + + /* Record tx info. */ + tx_info = &txq->tx_info[wqe_info.pi]; + tx_info->mbuf = mbuf_pkt; + tx_info->wqebb_cnt = wqe_info.wqebb_cnt; + + hinic3_prepare_sq_ctrl(&wqe_combo, &wqe_info); + + tx_bytes += mbuf_pkt->pkt_len; + } + + /* Update txq stats. */ + if (nb_tx) { + hinic3_write_db(txq->db_addr, txq->q_id, (int)(txq->cos), + SQ_CFLAG_DP, + MASKED_QUEUE_IDX(txq, txq->prod_idx)); + txq->txq_stats.packets += nb_tx; + txq->txq_stats.bytes += tx_bytes; + } + txq->txq_stats.burst_pkts = nb_tx; + +#ifdef HINIC3_XSTAT_PROF_TX + t2 = rte_get_tsc_cycles(); + txq->txq_stats.app_tsc = t1 - txq->prof_tx_end_tsc; + txq->prof_tx_end_tsc = t2; + txq->txq_stats.pmd_tsc = t2 - t1; + txq->txq_stats.burst_pkts = nb_tx; +#endif + + return nb_tx; +} + int hinic3_stop_sq(struct hinic3_txq *txq) { diff --git a/drivers/net/hinic3/hinic3_tx.h b/drivers/net/hinic3/hinic3_tx.h index f4c61ea1b1..6026b3fabc 100644 --- a/drivers/net/hinic3/hinic3_tx.h +++ b/drivers/net/hinic3/hinic3_tx.h @@ -308,6 +308,7 @@ struct __rte_cache_aligned hinic3_txq { void hinic3_flush_txqs(struct hinic3_nic_dev *nic_dev); void hinic3_free_txq_mbufs(struct hinic3_txq *txq); void hinic3_free_all_txq_mbufs(struct hinic3_nic_dev *nic_dev); +u16 hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, u16 nb_pkts); int hinic3_stop_sq(struct hinic3_txq *txq); int hinic3_start_all_sqs(struct rte_eth_dev *eth_dev); int hinic3_tx_done_cleanup(void *txq, uint32_t free_cnt); -- 2.47.0.windows.2