[PATCH v5 14/32] net/sssnic: support Rx queue setup and release

[<wanry@3snic.com>]

From: Renyong Wan <wanry@3snic.com>

Signed-off-by: Steven Song <steven.song@3snic.com>
Signed-off-by: Renyong Wan <wanry@3snic.com>
---
v2:
* Removed error.h from including files.
---
 drivers/net/sssnic/meson.build        |   1 +
 drivers/net/sssnic/sssnic_ethdev.c    |   4 +
 drivers/net/sssnic/sssnic_ethdev.h    |   2 +
 drivers/net/sssnic/sssnic_ethdev_rx.c | 420 ++++++++++++++++++++++++++
 drivers/net/sssnic/sssnic_ethdev_rx.h |  24 ++
 5 files changed, 451 insertions(+)
 create mode 100644 drivers/net/sssnic/sssnic_ethdev_rx.c
 create mode 100644 drivers/net/sssnic/sssnic_ethdev_rx.h

diff --git a/drivers/net/sssnic/meson.build b/drivers/net/sssnic/meson.build
index 12e967722a..7c3516a279 100644
--- a/drivers/net/sssnic/meson.build
+++ b/drivers/net/sssnic/meson.build
@@ -19,4 +19,5 @@ objs = [base_objs]
 sources = files(
         'sssnic_ethdev.c',
         'sssnic_ethdev_link.c',
+        'sssnic_ethdev_rx.c',
 )
diff --git a/drivers/net/sssnic/sssnic_ethdev.c b/drivers/net/sssnic/sssnic_ethdev.c
index 1e2b5c0450..f98510a55d 100644
--- a/drivers/net/sssnic/sssnic_ethdev.c
+++ b/drivers/net/sssnic/sssnic_ethdev.c
@@ -10,6 +10,7 @@
 #include "base/sssnic_api.h"
 #include "sssnic_ethdev.h"
 #include "sssnic_ethdev_link.h"
+#include "sssnic_ethdev_rx.h"
 
 static int
 sssnic_ethdev_infos_get(struct rte_eth_dev *ethdev,
@@ -335,6 +336,7 @@ sssnic_ethdev_release(struct rte_eth_dev *ethdev)
 	struct sssnic_hw *hw = SSSNIC_ETHDEV_TO_HW(ethdev);
 
 	sssnic_ethdev_link_intr_disable(ethdev);
+	sssnic_ethdev_rx_queue_all_release(ethdev);
 	sssnic_ethdev_mac_addrs_clean(ethdev);
 	sssnic_hw_shutdown(hw);
 	rte_free(hw);
@@ -350,6 +352,8 @@ static const struct eth_dev_ops sssnic_ethdev_ops = {
 	.mac_addr_remove = sssnic_ethdev_mac_addr_remove,
 	.mac_addr_add = sssnic_ethdev_mac_addr_add,
 	.set_mc_addr_list = sssnic_ethdev_set_mc_addr_list,
+	.rx_queue_setup = sssnic_ethdev_rx_queue_setup,
+	.rx_queue_release = sssnic_ethdev_rx_queue_release,
 };
 
 static int
diff --git a/drivers/net/sssnic/sssnic_ethdev.h b/drivers/net/sssnic/sssnic_ethdev.h
index 636b3fd04c..51740413c6 100644
--- a/drivers/net/sssnic/sssnic_ethdev.h
+++ b/drivers/net/sssnic/sssnic_ethdev.h
@@ -56,6 +56,8 @@
 #define SSSNIC_ETHDEV_MAX_NUM_UC_MAC 128
 #define SSSNIC_ETHDEV_MAX_NUM_MC_MAC 2048
 
+#define SSSNIC_ETHDEV_DEF_RX_FREE_THRESH 32
+
 struct sssnic_netdev {
 	void *hw;
 	struct rte_ether_addr *mcast_addrs;
diff --git a/drivers/net/sssnic/sssnic_ethdev_rx.c b/drivers/net/sssnic/sssnic_ethdev_rx.c
new file mode 100644
index 0000000000..0f489cf82f
--- /dev/null
+++ b/drivers/net/sssnic/sssnic_ethdev_rx.c
@@ -0,0 +1,420 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018-2022 Shenzhen 3SNIC Information Technology Co., Ltd.
+ */
+
+#include <rte_io.h>
+#include <rte_common.h>
+#include <rte_memzone.h>
+#include <ethdev_pci.h>
+
+#include "sssnic_log.h"
+#include "sssnic_ethdev.h"
+#include "sssnic_ethdev_rx.h"
+#include "base/sssnic_hw.h"
+#include "base/sssnic_workq.h"
+#include "base/sssnic_api.h"
+#include "base/sssnic_misc.h"
+
+/* hardware format of rx descriptor*/
+struct sssnic_ethdev_rx_desc {
+	/* status field */
+	union {
+		uint32_t dword0;
+		struct {
+			uint32_t ip_csum_err : 1;
+			uint32_t tcp_csum_err : 1;
+			uint32_t udp_csum_err : 1;
+			uint32_t igmp_csum_err : 1;
+			uint32_t icmpv4_csum_err : 1;
+			uint32_t icmpv6_csum_err : 1;
+			uint32_t sctp_crc_err : 1;
+			uint32_t hw_crc_err : 1;
+			uint32_t other_err : 1;
+			uint32_t err_resvd0 : 7;
+			uint32_t lro_num : 8;
+			uint32_t resvd1 : 1;
+			uint32_t lro_push : 1;
+			uint32_t lro_enter : 1;
+			uint32_t lro_intr : 1;
+			uint32_t flush : 1;
+			uint32_t decry : 1;
+			uint32_t bp_en : 1;
+			uint32_t done : 1;
+		};
+		struct {
+			uint32_t status_err : 16;
+			uint32_t status_rest : 16;
+		};
+	};
+
+	/* VLAN and length field */
+	union {
+		uint32_t dword1;
+		struct {
+			uint32_t vlan : 16;
+			uint32_t len : 16;
+		};
+	};
+
+	/* offload field */
+	union {
+		uint32_t dword2;
+		struct {
+			uint32_t pkt_type : 12;
+			uint32_t dword2_resvd0 : 9;
+			uint32_t vlan_en : 1;
+			uint32_t dword2_resvd1 : 2;
+			uint32_t rss_type : 8;
+		};
+	};
+
+	/* rss hash field */
+	union {
+		uint32_t dword3;
+		uint32_t rss_hash;
+	};
+
+	uint32_t dword4;
+	uint32_t dword5;
+	uint32_t dword6;
+	uint32_t dword7;
+} __rte_cache_aligned;
+
+struct sssnic_ethdev_rx_entry {
+	struct rte_mbuf *pktmbuf;
+};
+
+struct sssnic_ethdev_rxq {
+	struct rte_eth_dev *ethdev;
+	struct sssnic_workq *workq;
+	volatile struct sssnic_ethdev_rx_desc *desc;
+	const struct rte_memzone *pi_mz;
+	const struct rte_memzone *desc_mz;
+	struct rte_mempool *mp;
+	struct sssnic_ethdev_rx_entry *rxe;
+	volatile uint16_t *hw_pi_addr;
+	uint8_t *doorbell;
+	struct sssnic_ethdev_rxq_stats stats;
+	uint16_t port;
+	uint16_t qid;
+	uint16_t depth;
+	uint16_t rx_buf_size;
+	uint16_t rx_free_thresh;
+	struct {
+		uint16_t enable : 1;
+		uint16_t msix_id : 15;
+	} intr;
+	uint32_t resvd0;
+} __rte_cache_aligned;
+
+/* Hardware format of rxq entry */
+struct sssnic_ethdev_rxq_entry {
+	uint32_t buf_hi_addr;
+	uint32_t buf_lo_addr;
+	uint32_t desc_hi_addr;
+	uint32_t desc_lo_addr;
+};
+
+#define SSSNIC_ETHDEV_RX_ENTRY_SZ_BITS 4
+#define SSSNIC_ETHDEV_RXQ_ENTRY_SZ (RTE_BIT32(SSSNIC_ETHDEV_RX_ENTRY_SZ_BITS))
+
+#define SSSNIC_ETHDEV_RXQ_ENTRY(rxq, idx)                                      \
+	SSSNIC_WORKQ_ENTRY_CAST((rxq)->workq, idx,                             \
+		struct sssnic_ethdev_rxq_entry)
+
+static const uint16_t sssnic_ethdev_rx_buf_size_tbl[] = { 32, 64, 96, 128, 192,
+	256, 384, 512, 768, 1024, 1536, 2048, 3072, 4096, 8192, 16384 };
+
+#define SSSNIC_ETHDEV_RX_BUF_SIZE_COUNT (RTE_DIM(sssnic_ethdev_rx_buf_size_tbl))
+
+#define SSSNIC_ETHDEV_MIN_RX_BUF_SIZE (sssnic_ethdev_rx_buf_size_tbl[0])
+#define SSSNIC_ETHDEV_MAX_RX_BUF_SIZE                                          \
+	(sssnic_ethdev_rx_buf_size_tbl[SSSNIC_ETHDEV_RX_BUF_SIZE_COUNT - 1])
+
+#define SSSNIC_ETHDEV_DEF_RX_BUF_SIZE_IDX 11 /* 2048 Bytes */
+
+/* Doorbell offset 8192 */
+#define SSSNIC_ETHDEV_RXQ_DB_OFFSET 0x2000
+
+struct sssnic_ethdev_rxq_doorbell {
+	union {
+		uint64_t u64;
+		struct {
+			union {
+				uint32_t dword0;
+				struct {
+					uint32_t qid : 13;
+					uint32_t resvd0 : 9;
+					uint32_t nf : 1;
+					uint32_t cf : 1;
+					uint32_t cos : 3;
+					uint32_t service : 5;
+				};
+			};
+			union {
+				uint32_t dword1;
+				struct {
+					uint32_t pi_hi : 8;
+					uint32_t resvd1 : 24;
+				};
+			};
+		};
+	};
+};
+
+static inline uint16_t
+sssnic_ethdev_rxq_num_used_entries(struct sssnic_ethdev_rxq *rxq)
+{
+	return sssnic_workq_num_used_entries(rxq->workq);
+}
+
+static inline uint16_t
+sssnic_ethdev_rxq_ci_get(struct sssnic_ethdev_rxq *rxq)
+{
+	return sssnic_workq_ci_get(rxq->workq);
+}
+
+static inline void
+sssnic_ethdev_rxq_consume(struct sssnic_ethdev_rxq *rxq, uint16_t num_entries)
+{
+	sssnic_workq_consume_fast(rxq->workq, num_entries);
+}
+
+static void
+sssnic_ethdev_rx_buf_size_optimize(uint32_t orig_size, uint16_t *new_size)
+{
+	uint32_t i;
+	uint16_t size;
+
+	if (orig_size >= SSSNIC_ETHDEV_MAX_RX_BUF_SIZE) {
+		*new_size = SSSNIC_ETHDEV_MAX_RX_BUF_SIZE;
+		return;
+	}
+
+	size = SSSNIC_ETHDEV_MIN_RX_BUF_SIZE;
+	for (i = 0; i < SSSNIC_ETHDEV_RX_BUF_SIZE_COUNT; i++) {
+		if (orig_size == sssnic_ethdev_rx_buf_size_tbl[i]) {
+			*new_size = sssnic_ethdev_rx_buf_size_tbl[i];
+			return;
+		}
+
+		if (orig_size < sssnic_ethdev_rx_buf_size_tbl[i]) {
+			*new_size = size;
+			return;
+		}
+		size = sssnic_ethdev_rx_buf_size_tbl[i];
+	}
+	*new_size = size;
+}
+
+static void
+sssnic_ethdev_rxq_entries_init(struct sssnic_ethdev_rxq *rxq)
+{
+	struct sssnic_ethdev_rxq_entry *rqe;
+	rte_iova_t rxd_iova;
+	int i;
+
+	rxd_iova = rxq->desc_mz->iova;
+
+	for (i = 0; i < rxq->depth; i++) {
+		rqe = SSSNIC_ETHDEV_RXQ_ENTRY(rxq, i);
+		rqe->desc_hi_addr = SSSNIC_UPPER_32_BITS(rxd_iova);
+		rqe->desc_lo_addr = SSSNIC_LOWER_32_BITS(rxd_iova);
+		rxd_iova += sizeof(struct sssnic_ethdev_rx_desc);
+	}
+}
+
+int
+sssnic_ethdev_rx_queue_setup(struct rte_eth_dev *ethdev, uint16_t rx_queue_id,
+	uint16_t nb_rx_desc, unsigned int socket_id,
+	const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool)
+{
+	int ret;
+	struct sssnic_hw *hw;
+	struct sssnic_ethdev_rxq *rxq;
+	uint16_t q_depth;
+	uint16_t rx_buf_size;
+	uint16_t rx_free_thresh;
+	char m_name[RTE_MEMZONE_NAMESIZE];
+
+	hw = SSSNIC_ETHDEV_TO_HW(ethdev);
+
+	q_depth = nb_rx_desc;
+	/* Adjust q_depth to power of 2 */
+	if (!rte_is_power_of_2(nb_rx_desc)) {
+		q_depth = 1 << rte_log2_u32(nb_rx_desc);
+		PMD_DRV_LOG(NOTICE,
+			"nb_rx_desc(%u) is not power of 2, adjust to %u",
+			nb_rx_desc, q_depth);
+	}
+
+	if (q_depth > SSSNIC_ETHDEV_MAX_NUM_Q_DESC) {
+		PMD_DRV_LOG(ERR, "nb_rx_desc(%u) is out of range(max. %u)",
+			q_depth, SSSNIC_ETHDEV_MAX_NUM_Q_DESC);
+		return -EINVAL;
+	}
+
+	rx_buf_size =
+		rte_pktmbuf_data_room_size(mb_pool) - RTE_PKTMBUF_HEADROOM;
+	if (rx_buf_size < SSSNIC_ETHDEV_MIN_RX_BUF_SIZE) {
+		PMD_DRV_LOG(ERR,
+			"Bad data_room_size(%u), must be great than %u",
+			rte_pktmbuf_data_room_size(mb_pool),
+			RTE_PKTMBUF_HEADROOM + SSSNIC_ETHDEV_MIN_RX_BUF_SIZE);
+		return -EINVAL;
+	}
+	sssnic_ethdev_rx_buf_size_optimize(rx_buf_size, &rx_buf_size);
+
+	if (rx_conf->rx_free_thresh > 0)
+		rx_free_thresh = rx_conf->rx_free_thresh;
+	else
+		rx_free_thresh = SSSNIC_ETHDEV_DEF_RX_FREE_THRESH;
+	if (rx_free_thresh >= q_depth - 1) {
+		PMD_DRV_LOG(ERR,
+			"rx_free_thresh(%u) must be less than nb_rx_desc(%u)-1",
+			rx_free_thresh, q_depth);
+		return -EINVAL;
+	}
+
+	snprintf(m_name, sizeof(m_name), "sssnic_p%u_rq%u",
+		ethdev->data->port_id, rx_queue_id);
+
+	rxq = rte_zmalloc_socket(m_name, sizeof(struct sssnic_ethdev_rxq),
+		RTE_CACHE_LINE_SIZE, (int)socket_id);
+	if (rxq == NULL) {
+		PMD_DRV_LOG(ERR,
+			"Failed to allocate memory for sssnic port %u, rxq %u",
+			ethdev->data->port_id, rx_queue_id);
+		return -ENOMEM;
+	}
+
+	rxq->ethdev = ethdev;
+	rxq->mp = mb_pool;
+	rxq->doorbell = hw->db_base_addr + SSSNIC_ETHDEV_RXQ_DB_OFFSET;
+	rxq->port = ethdev->data->port_id;
+	rxq->qid = rx_queue_id;
+	rxq->depth = q_depth;
+	rxq->rx_buf_size = rx_buf_size;
+	rxq->rx_free_thresh = rx_free_thresh;
+
+	snprintf(m_name, sizeof(m_name), "sssnic_p%u_rq%u_wq",
+		ethdev->data->port_id, rx_queue_id);
+	rxq->workq = sssnic_workq_new(m_name, (int)socket_id,
+		SSSNIC_ETHDEV_RXQ_ENTRY_SZ, q_depth);
+	if (rxq->workq == NULL) {
+		PMD_DRV_LOG(ERR,
+			"Failed to create workq for sssnic port %u, rxq %u",
+			ethdev->data->port_id, rx_queue_id);
+		ret = -ENOMEM;
+		goto new_workq_fail;
+	}
+
+	rxq->pi_mz = rte_eth_dma_zone_reserve(ethdev, "sssnic_rxpi_mz",
+		rxq->qid, RTE_PGSIZE_4K, RTE_CACHE_LINE_SIZE, (int)socket_id);
+	if (rxq->pi_mz == NULL) {
+		PMD_DRV_LOG(ERR,
+			"Failed to alloc DMA memory for rx pi of sssnic port %u rxq %u",
+			ethdev->data->port_id, rx_queue_id);
+		ret = -ENOMEM;
+		goto alloc_pi_mz_fail;
+	}
+	rxq->hw_pi_addr = (uint16_t *)rxq->pi_mz->addr;
+
+	rxq->desc_mz = rte_eth_dma_zone_reserve(ethdev, "sssnic_rxd_mz",
+		rxq->qid, sizeof(struct sssnic_ethdev_rx_desc) * rxq->depth,
+		RTE_CACHE_LINE_SIZE, (int)socket_id);
+	if (rxq->pi_mz == NULL) {
+		PMD_DRV_LOG(ERR,
+			"Failed to alloc DMA memory for rx desc of sssnic port %u rxq %u",
+			ethdev->data->port_id, rx_queue_id);
+		ret = -ENOMEM;
+		goto alloc_rxd_mz_fail;
+	}
+	rxq->desc = (struct sssnic_ethdev_rx_desc *)rxq->desc_mz->addr;
+
+	snprintf(m_name, sizeof(m_name), "sssnic_p%u_rq%u_rxe",
+		ethdev->data->port_id, rx_queue_id);
+
+	rxq->rxe = rte_zmalloc_socket(m_name,
+		sizeof(struct sssnic_ethdev_rx_entry) * rxq->depth,
+		RTE_CACHE_LINE_SIZE, (int)socket_id);
+	if (rxq->rxe == NULL) {
+		PMD_DRV_LOG(ERR,
+			"Failed to alloc memory for rx entries of sssnic port %u rxq %u",
+			ethdev->data->port_id, rx_queue_id);
+		ret = -ENOMEM;
+		goto alloc_pktmbuf_fail;
+	}
+
+	sssnic_ethdev_rxq_entries_init(rxq);
+
+	ethdev->data->rx_queues[rx_queue_id] = rxq;
+
+	return 0;
+
+alloc_pktmbuf_fail:
+	rte_memzone_free(rxq->desc_mz);
+alloc_rxd_mz_fail:
+	rte_memzone_free(rxq->pi_mz);
+alloc_pi_mz_fail:
+	sssnic_workq_destroy(rxq->workq);
+new_workq_fail:
+	rte_free(rxq);
+
+	return ret;
+}
+
+static void
+sssnic_ethdev_rxq_pktmbufs_release(struct sssnic_ethdev_rxq *rxq)
+{
+	struct sssnic_ethdev_rx_entry *rxe;
+	volatile struct sssnic_ethdev_rx_desc *rxd;
+	uint16_t num_entries;
+	uint16_t ci;
+	uint16_t i;
+
+	num_entries = sssnic_ethdev_rxq_num_used_entries(rxq);
+	for (i = 0; i < num_entries; i++) {
+		ci = sssnic_ethdev_rxq_ci_get(rxq);
+		rxd = &rxq->desc[ci];
+		rxd->dword0 = 0;
+		rxe = &rxq->rxe[ci];
+		rte_pktmbuf_free(rxe->pktmbuf);
+		rxe->pktmbuf = NULL;
+		sssnic_ethdev_rxq_consume(rxq, 1);
+	}
+}
+
+static void
+sssnic_ethdev_rxq_free(struct sssnic_ethdev_rxq *rxq)
+{
+	if (rxq == NULL)
+		return;
+
+	sssnic_ethdev_rxq_pktmbufs_release(rxq);
+	rte_free(rxq->rxe);
+	rte_memzone_free(rxq->desc_mz);
+	rte_memzone_free(rxq->pi_mz);
+	sssnic_workq_destroy(rxq->workq);
+	rte_free(rxq);
+}
+
+void
+sssnic_ethdev_rx_queue_release(struct rte_eth_dev *ethdev, uint16_t queue_id)
+{
+	struct sssnic_ethdev_rxq *rxq = ethdev->data->rx_queues[queue_id];
+
+	if (rxq == NULL)
+		return;
+	sssnic_ethdev_rxq_free(rxq);
+	ethdev->data->rx_queues[queue_id] = NULL;
+}
+
+void
+sssnic_ethdev_rx_queue_all_release(struct rte_eth_dev *ethdev)
+{
+	uint16_t qid;
+
+	for (qid = 0; qid < ethdev->data->nb_rx_queues; qid++)
+		sssnic_ethdev_rx_queue_release(ethdev, qid);
+}
diff --git a/drivers/net/sssnic/sssnic_ethdev_rx.h b/drivers/net/sssnic/sssnic_ethdev_rx.h
new file mode 100644
index 0000000000..dc41811a2f
--- /dev/null
+++ b/drivers/net/sssnic/sssnic_ethdev_rx.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018-2022 Shenzhen 3SNIC Information Technology Co., Ltd.
+ */
+
+#ifndef _SSSNIC_ETHDEV_RX_H_
+#define _SSSNIC_ETHDEV_RX_H_
+
+struct sssnic_ethdev_rxq_stats {
+	uint64_t packets;
+	uint64_t bytes;
+	uint64_t csum_errors;
+	uint64_t other_errors;
+	uint64_t nombuf;
+	uint64_t burst;
+};
+
+int sssnic_ethdev_rx_queue_setup(struct rte_eth_dev *ethdev,
+	uint16_t rx_queue_id, uint16_t nb_rx_desc, unsigned int socket_id,
+	const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool);
+void sssnic_ethdev_rx_queue_release(struct rte_eth_dev *ethdev,
+	uint16_t queue_id);
+void sssnic_ethdev_rx_queue_all_release(struct rte_eth_dev *ethdev);
+
+#endif
-- 
2.27.0