From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id E7486A00C4; Mon, 31 Oct 2022 05:08:21 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id C923642825; Mon, 31 Oct 2022 05:07:33 +0100 (CET) Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by mails.dpdk.org (Postfix) with ESMTP id 578E840A8B for ; Mon, 31 Oct 2022 05:07:25 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1667189245; x=1698725245; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=X5780iKEyX4idL1oA45PI2Ib31V4TojLXr3JkPW1CSA=; b=c3rrMJ899zrLdXCyr9R9wdqAOrSt0B6dIuccibTikKlhV4yuLoQKkPYF DH204Uv0cvL7rSsXE9zDld4nAhUcYnqCl5oY1cSi4ot8BCfl9qeFgLnaX 7SMXl749iUn4nnzau3xbkHizn3DaB1KsNEJ4W7R/ZRSuPvVShDs5ehugt NQ9COh5vhLYOxxiwKmPwe6Q1GTqvgDpZtGbamkTyHgxNuIhffuowHPO7E pyY8C51p3ANTDOrrxTf1hF7BZihg1hvqfIfFC/eikADbUBDsT87GZlNbc weHUOJzxWky+C4XgjmWzZltBnZqZ2C88uuCWgStG9SqtXJU8XnVA8jb0h g==; X-IronPort-AV: E=McAfee;i="6500,9779,10516"; a="310508293" X-IronPort-AV: E=Sophos;i="5.95,227,1661842800"; d="scan'208";a="310508293" Received: from orsmga003.jf.intel.com ([10.7.209.27]) by orsmga103.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 30 Oct 2022 21:07:24 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10516"; a="584527089" X-IronPort-AV: E=Sophos;i="5.95,227,1661842800"; d="scan'208";a="584527089" Received: from dpdk-beileix-3.sh.intel.com ([10.67.110.253]) by orsmga003.jf.intel.com with ESMTP; 30 Oct 2022 21:07:22 -0700 From: beilei.xing@intel.com To: andrew.rybchenko@oktetlabs.ru, jingjing.wu@intel.com, beilei.xing@intel.com Cc: dev@dpdk.org, Junfeng Guo , Xiaoyun Li Subject: [PATCH v16 10/18] net/idpf: add support for basic Rx datapath Date: Mon, 31 Oct 2022 03:36:43 +0000 Message-Id: <20221031033651.98072-11-beilei.xing@intel.com> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20221031033651.98072-1-beilei.xing@intel.com> References: <20221029032729.22772-1-beilei.xing@intel.com> <20221031033651.98072-1-beilei.xing@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Junfeng Guo Add basic Rx support in split queue mode and single queue mode. Signed-off-by: Beilei Xing Signed-off-by: Xiaoyun Li Signed-off-by: Junfeng Guo --- drivers/net/idpf/idpf_ethdev.c | 2 + drivers/net/idpf/idpf_rxtx.c | 273 +++++++++++++++++++++++++++++++++ drivers/net/idpf/idpf_rxtx.h | 7 +- 3 files changed, 281 insertions(+), 1 deletion(-) diff --git a/drivers/net/idpf/idpf_ethdev.c b/drivers/net/idpf/idpf_ethdev.c index 856f3d7266..2f1f9eeee5 100644 --- a/drivers/net/idpf/idpf_ethdev.c +++ b/drivers/net/idpf/idpf_ethdev.c @@ -348,6 +348,8 @@ idpf_dev_start(struct rte_eth_dev *dev) goto err_mtu; } + idpf_set_rx_function(dev); + ret = idpf_vc_ena_dis_vport(vport, true); if (ret != 0) { PMD_DRV_LOG(ERR, "Failed to enable vport"); diff --git a/drivers/net/idpf/idpf_rxtx.c b/drivers/net/idpf/idpf_rxtx.c index 053409b99a..ea499c4d37 100644 --- a/drivers/net/idpf/idpf_rxtx.c +++ b/drivers/net/idpf/idpf_rxtx.c @@ -1208,3 +1208,276 @@ idpf_stop_queues(struct rte_eth_dev *dev) PMD_DRV_LOG(WARNING, "Fail to stop Tx queue %d", i); } } + +static void +idpf_split_rx_bufq_refill(struct idpf_rx_queue *rx_bufq) +{ + volatile struct virtchnl2_splitq_rx_buf_desc *rx_buf_ring; + volatile struct virtchnl2_splitq_rx_buf_desc *rx_buf_desc; + uint16_t nb_refill = rx_bufq->rx_free_thresh; + uint16_t nb_desc = rx_bufq->nb_rx_desc; + uint16_t next_avail = rx_bufq->rx_tail; + struct rte_mbuf *nmb[rx_bufq->rx_free_thresh]; + struct rte_eth_dev *dev; + uint64_t dma_addr; + uint16_t delta; + int i; + + if (rx_bufq->nb_rx_hold < rx_bufq->rx_free_thresh) + return; + + rx_buf_ring = rx_bufq->rx_ring; + delta = nb_desc - next_avail; + if (unlikely(delta < nb_refill)) { + if (likely(rte_pktmbuf_alloc_bulk(rx_bufq->mp, nmb, delta) == 0)) { + for (i = 0; i < delta; i++) { + rx_buf_desc = &rx_buf_ring[next_avail + i]; + rx_bufq->sw_ring[next_avail + i] = nmb[i]; + dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb[i])); + rx_buf_desc->hdr_addr = 0; + rx_buf_desc->pkt_addr = dma_addr; + } + nb_refill -= delta; + next_avail = 0; + rx_bufq->nb_rx_hold -= delta; + } else { + dev = &rte_eth_devices[rx_bufq->port_id]; + dev->data->rx_mbuf_alloc_failed += nb_desc - next_avail; + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u queue_id=%u", + rx_bufq->port_id, rx_bufq->queue_id); + return; + } + } + + if (nb_desc - next_avail >= nb_refill) { + if (likely(rte_pktmbuf_alloc_bulk(rx_bufq->mp, nmb, nb_refill) == 0)) { + for (i = 0; i < nb_refill; i++) { + rx_buf_desc = &rx_buf_ring[next_avail + i]; + rx_bufq->sw_ring[next_avail + i] = nmb[i]; + dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb[i])); + rx_buf_desc->hdr_addr = 0; + rx_buf_desc->pkt_addr = dma_addr; + } + next_avail += nb_refill; + rx_bufq->nb_rx_hold -= nb_refill; + } else { + dev = &rte_eth_devices[rx_bufq->port_id]; + dev->data->rx_mbuf_alloc_failed += nb_desc - next_avail; + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u queue_id=%u", + rx_bufq->port_id, rx_bufq->queue_id); + } + } + + IDPF_PCI_REG_WRITE(rx_bufq->qrx_tail, next_avail); + + rx_bufq->rx_tail = next_avail; +} + +uint16_t +idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc_ring; + volatile struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; + uint16_t pktlen_gen_bufq_id; + struct idpf_rx_queue *rxq; + struct rte_mbuf *rxm; + uint16_t rx_id_bufq1; + uint16_t rx_id_bufq2; + uint16_t pkt_len; + uint16_t bufq_id; + uint16_t gen_id; + uint16_t rx_id; + uint16_t nb_rx; + + nb_rx = 0; + rxq = rx_queue; + + if (unlikely(rxq == NULL) || unlikely(!rxq->q_started)) + return nb_rx; + + rx_id = rxq->rx_tail; + rx_id_bufq1 = rxq->bufq1->rx_next_avail; + rx_id_bufq2 = rxq->bufq2->rx_next_avail; + rx_desc_ring = rxq->rx_ring; + + while (nb_rx < nb_pkts) { + rx_desc = &rx_desc_ring[rx_id]; + + pktlen_gen_bufq_id = + rte_le_to_cpu_16(rx_desc->pktlen_gen_bufq_id); + gen_id = (pktlen_gen_bufq_id & + VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M) >> + VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_S; + if (gen_id != rxq->expected_gen_id) + break; + + pkt_len = (pktlen_gen_bufq_id & + VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_M) >> + VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_S; + if (pkt_len == 0) + PMD_RX_LOG(ERR, "Packet length is 0"); + + rx_id++; + if (unlikely(rx_id == rxq->nb_rx_desc)) { + rx_id = 0; + rxq->expected_gen_id ^= 1; + } + + bufq_id = (pktlen_gen_bufq_id & + VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_M) >> + VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_S; + if (bufq_id == 0) { + rxm = rxq->bufq1->sw_ring[rx_id_bufq1]; + rx_id_bufq1++; + if (unlikely(rx_id_bufq1 == rxq->bufq1->nb_rx_desc)) + rx_id_bufq1 = 0; + rxq->bufq1->nb_rx_hold++; + } else { + rxm = rxq->bufq2->sw_ring[rx_id_bufq2]; + rx_id_bufq2++; + if (unlikely(rx_id_bufq2 == rxq->bufq2->nb_rx_desc)) + rx_id_bufq2 = 0; + rxq->bufq2->nb_rx_hold++; + } + + rxm->pkt_len = pkt_len; + rxm->data_len = pkt_len; + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rxm->next = NULL; + rxm->nb_segs = 1; + rxm->port = rxq->port_id; + + rx_pkts[nb_rx++] = rxm; + } + + if (nb_rx > 0) { + rxq->rx_tail = rx_id; + if (rx_id_bufq1 != rxq->bufq1->rx_next_avail) + rxq->bufq1->rx_next_avail = rx_id_bufq1; + if (rx_id_bufq2 != rxq->bufq2->rx_next_avail) + rxq->bufq2->rx_next_avail = rx_id_bufq2; + + idpf_split_rx_bufq_refill(rxq->bufq1); + idpf_split_rx_bufq_refill(rxq->bufq2); + } + + return nb_rx; +} + +static inline void +idpf_update_rx_tail(struct idpf_rx_queue *rxq, uint16_t nb_hold, + uint16_t rx_id) +{ + nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); + + if (nb_hold > rxq->rx_free_thresh) { + PMD_RX_LOG(DEBUG, + "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u", + rxq->port_id, rxq->queue_id, rx_id, nb_hold); + rx_id = (uint16_t)((rx_id == 0) ? + (rxq->nb_rx_desc - 1) : (rx_id - 1)); + IDPF_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + nb_hold = 0; + } + rxq->nb_rx_hold = nb_hold; +} + +uint16_t +idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts) +{ + volatile union virtchnl2_rx_desc *rx_ring; + volatile union virtchnl2_rx_desc *rxdp; + union virtchnl2_rx_desc rxd; + struct idpf_rx_queue *rxq; + uint16_t rx_id, nb_hold; + struct rte_eth_dev *dev; + uint16_t rx_packet_len; + struct rte_mbuf *rxm; + struct rte_mbuf *nmb; + uint16_t rx_status0; + uint64_t dma_addr; + uint16_t nb_rx; + + nb_rx = 0; + nb_hold = 0; + rxq = rx_queue; + + if (unlikely(rxq == NULL) || unlikely(!rxq->q_started)) + return nb_rx; + + rx_id = rxq->rx_tail; + rx_ring = rxq->rx_ring; + + while (nb_rx < nb_pkts) { + rxdp = &rx_ring[rx_id]; + rx_status0 = rte_le_to_cpu_16(rxdp->flex_nic_wb.status_error0); + + /* Check the DD bit first */ + if ((rx_status0 & (1 << VIRTCHNL2_RX_FLEX_DESC_STATUS0_DD_S)) == 0) + break; + + nmb = rte_mbuf_raw_alloc(rxq->mp); + if (unlikely(nmb == NULL)) { + dev = &rte_eth_devices[rxq->port_id]; + dev->data->rx_mbuf_alloc_failed++; + PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " + "queue_id=%u", rxq->port_id, rxq->queue_id); + break; + } + rxd = *rxdp; /* copy descriptor in ring to temp variable*/ + + nb_hold++; + rxm = rxq->sw_ring[rx_id]; + rxq->sw_ring[rx_id] = nmb; + rx_id++; + if (unlikely(rx_id == rxq->nb_rx_desc)) + rx_id = 0; + + /* Prefetch next mbuf */ + rte_prefetch0(rxq->sw_ring[rx_id]); + + /* When next RX descriptor is on a cache line boundary, + * prefetch the next 4 RX descriptors and next 8 pointers + * to mbufs. + */ + if ((rx_id & 0x3) == 0) { + rte_prefetch0(&rx_ring[rx_id]); + rte_prefetch0(rxq->sw_ring[rx_id]); + } + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); + rxdp->read.hdr_addr = 0; + rxdp->read.pkt_addr = dma_addr; + + rx_packet_len = (rte_cpu_to_le_16(rxd.flex_nic_wb.pkt_len) & + VIRTCHNL2_RX_FLEX_DESC_PKT_LEN_M); + + rxm->data_off = RTE_PKTMBUF_HEADROOM; + rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); + rxm->nb_segs = 1; + rxm->next = NULL; + rxm->pkt_len = rx_packet_len; + rxm->data_len = rx_packet_len; + rxm->port = rxq->port_id; + + rx_pkts[nb_rx++] = rxm; + } + rxq->rx_tail = rx_id; + + idpf_update_rx_tail(rxq, nb_hold, rx_id); + + return nb_rx; +} + +void +idpf_set_rx_function(struct rte_eth_dev *dev) +{ + struct idpf_vport *vport = dev->data->dev_private; + + if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) + dev->rx_pkt_burst = idpf_splitq_recv_pkts; + else + dev->rx_pkt_burst = idpf_singleq_recv_pkts; +} diff --git a/drivers/net/idpf/idpf_rxtx.h b/drivers/net/idpf/idpf_rxtx.h index 37944edcac..650c6c1c3a 100644 --- a/drivers/net/idpf/idpf_rxtx.h +++ b/drivers/net/idpf/idpf_rxtx.h @@ -133,6 +133,11 @@ int idpf_tx_queue_init(struct rte_eth_dev *dev, uint16_t tx_queue_id); int idpf_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id); int idpf_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id); void idpf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid); - +uint16_t idpf_singleq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); +uint16_t idpf_splitq_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, + uint16_t nb_pkts); void idpf_stop_queues(struct rte_eth_dev *dev); + +void idpf_set_rx_function(struct rte_eth_dev *dev); #endif /* _IDPF_RXTX_H_ */ -- 2.26.2