[PATCH v5 04/32] net/sssnic: initialize hardware base

[<wanry@3snic.com>]

From: Renyong Wan <wanry@3snic.com>

Initializing hardware base make hardware ready to be access.

Signed-off-by: Steven Song <steven.song@3snic.com>
Signed-off-by: Renyong Wan <wanry@3snic.com>
---
 drivers/net/sssnic/base/meson.build  |  13 ++
 drivers/net/sssnic/base/sssnic_hw.c  | 207 +++++++++++++++++++++++++++
 drivers/net/sssnic/base/sssnic_hw.h  |  49 +++++++
 drivers/net/sssnic/base/sssnic_reg.h | 169 ++++++++++++++++++++++
 drivers/net/sssnic/meson.build       |   3 +
 drivers/net/sssnic/sssnic_ethdev.c   |  46 +++++-
 drivers/net/sssnic/sssnic_ethdev.h   |  18 +++
 7 files changed, 501 insertions(+), 4 deletions(-)
 create mode 100644 drivers/net/sssnic/base/meson.build
 create mode 100644 drivers/net/sssnic/base/sssnic_hw.c
 create mode 100644 drivers/net/sssnic/base/sssnic_reg.h
 create mode 100644 drivers/net/sssnic/sssnic_ethdev.h

diff --git a/drivers/net/sssnic/base/meson.build b/drivers/net/sssnic/base/meson.build
new file mode 100644
index 0000000000..3e64112c72
--- /dev/null
+++ b/drivers/net/sssnic/base/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018-2022 Shenzhen 3SNIC Information Technology Co., Ltd.
+
+sources = [
+        'sssnic_hw.c',
+]
+
+c_args = cflags
+base_lib = static_library('sssnic_base', sources,
+	dependencies: [static_rte_eal, static_rte_ethdev, static_rte_bus_pci, static_rte_net],
+	c_args: c_args)
+
+base_objs = base_lib.extract_all_objects()
diff --git a/drivers/net/sssnic/base/sssnic_hw.c b/drivers/net/sssnic/base/sssnic_hw.c
new file mode 100644
index 0000000000..8b7bba7644
--- /dev/null
+++ b/drivers/net/sssnic/base/sssnic_hw.c
@@ -0,0 +1,207 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018-2022 Shenzhen 3SNIC Information Technology Co., Ltd.
+ */
+
+#include <rte_byteorder.h>
+#include <rte_cycles.h>
+#include <ethdev_pci.h>
+
+#include "../sssnic_log.h"
+#include "sssnic_hw.h"
+#include "sssnic_reg.h"
+
+static int
+wait_for_sssnic_hw_ready(struct sssnic_hw *hw)
+{
+	struct sssnic_attr1_reg reg;
+	uint32_t timeout_ms = 10;
+
+	do {
+		reg.u32 = sssnic_cfg_reg_read(hw, SSSNIC_ATTR1_REG);
+		if (reg.u32 != 0xffffffff && reg.mgmt_init_status != 0)
+			return 0;
+		rte_delay_ms(1);
+	} while (--timeout_ms);
+
+	return -EBUSY;
+}
+
+static int
+wait_for_sssnic_db_enabled(struct sssnic_hw *hw)
+{
+	struct sssnic_attr4_reg r4;
+	struct sssnic_attr5_reg r5;
+	uint32_t timeout_ms = 60000;
+
+	do {
+		r4.u32 = sssnic_cfg_reg_read(hw, SSSNIC_ATTR4_REG);
+		r5.u32 = sssnic_cfg_reg_read(hw, SSSNIC_ATTR5_REG);
+		if (r4.db_ctrl == SSSNIC_DB_CTRL_ENABLE &&
+			r5.outbound_ctrl == SSSNIC_DB_CTRL_ENABLE)
+			return 0;
+		rte_delay_ms(1);
+	} while (--timeout_ms);
+
+	return -EBUSY;
+}
+
+static void
+sssnic_attr_setup(struct sssnic_hw *hw)
+{
+	struct sssnic_attr0_reg attr0;
+	struct sssnic_attr1_reg attr1;
+	struct sssnic_attr2_reg attr2;
+	struct sssnic_attr3_reg attr3;
+	struct sssnic_hw_attr *attr = &hw->attr;
+
+	attr0.u32 = sssnic_cfg_reg_read(hw, SSSNIC_ATTR0_REG);
+	attr1.u32 = sssnic_cfg_reg_read(hw, SSSNIC_ATTR1_REG);
+	attr2.u32 = sssnic_cfg_reg_read(hw, SSSNIC_ATTR2_REG);
+	attr3.u32 = sssnic_cfg_reg_read(hw, SSSNIC_ATTR3_REG);
+
+	attr->func_idx = attr0.func_idx;
+	attr->pf_idx = attr0.pf_idx;
+	attr->pci_idx = attr0.pci_idx;
+	attr->vf_off = attr0.vf_off;
+	attr->func_type = attr0.func_type;
+	attr->af_idx = attr1.af_idx;
+	attr->num_aeq = RTE_BIT32(attr1.num_aeq);
+	attr->num_ceq = attr2.num_ceq;
+	attr->num_irq = attr2.num_irq;
+	attr->global_vf_off = attr3.global_vf_off;
+
+	PMD_DRV_LOG(DEBUG, "attr0=0x%x, attr1=0x%x, attr2=0x%x, attr3=0x%x",
+		attr0.u32, attr1.u32, attr2.u32, attr3.u32);
+}
+
+/* AF and MF election */
+static void
+sssnic_af_setup(struct sssnic_hw *hw)
+{
+	struct sssnic_af_election_reg reg0;
+	struct sssnic_mf_election_reg reg1;
+
+	/* AF election */
+	reg0.u32 = sssnic_mgmt_reg_read(hw, SSSNIC_AF_ELECTION_REG);
+	reg0.func_idx = hw->attr.func_idx;
+	sssnic_mgmt_reg_write(hw, SSSNIC_AF_ELECTION_REG, reg0.u32);
+	reg0.u32 = sssnic_mgmt_reg_read(hw, SSSNIC_AF_ELECTION_REG);
+	hw->attr.af_idx = reg0.func_idx;
+	if (hw->attr.af_idx == hw->attr.func_idx) {
+		hw->attr.func_type = SSSNIC_FUNC_TYPE_AF;
+		PMD_DRV_LOG(INFO, "Elected PF %d as AF", hw->attr.func_idx);
+
+		/* MF election */
+		reg1.u32 = sssnic_mgmt_reg_read(hw, SSSNIC_MF_ELECTION_REG);
+		reg1.func_idx = hw->attr.func_idx;
+		sssnic_mgmt_reg_write(hw, SSSNIC_MF_ELECTION_REG, reg1.u32);
+		reg1.u32 = sssnic_mgmt_reg_read(hw, SSSNIC_MF_ELECTION_REG);
+		hw->attr.mf_idx = reg1.func_idx;
+		if (hw->attr.mf_idx == hw->attr.func_idx)
+			PMD_DRV_LOG(INFO, "Elected PF %d as MF",
+				hw->attr.func_idx);
+	}
+}
+
+void
+sssnic_msix_state_set(struct sssnic_hw *hw, uint16_t msix_id, int state)
+{
+	struct sssnic_msix_ctrl_reg reg;
+
+	reg.u32 = 0;
+	if (state == SSSNIC_MSIX_ENABLE)
+		reg.int_msk_clr = 1;
+	else
+		reg.int_msk_set = 1;
+	reg.msxi_idx = msix_id;
+	sssnic_cfg_reg_write(hw, SSSNIC_MSIX_CTRL_REG, reg.u32);
+}
+
+static void
+sssnic_msix_all_disable(struct sssnic_hw *hw)
+{
+	uint16_t i;
+	int num_irqs = hw->attr.num_irq;
+
+	for (i = 0; i < num_irqs; i++)
+		sssnic_msix_state_set(hw, i, SSSNIC_MSIX_DISABLE);
+}
+
+static void
+sssnic_pf_status_set(struct sssnic_hw *hw, enum sssnic_pf_status status)
+{
+	struct sssnic_attr6_reg reg;
+
+	reg.u32 = sssnic_cfg_reg_read(hw, SSSNIC_ATTR6_REG);
+	reg.pf_status = status;
+	sssnic_cfg_reg_write(hw, SSSNIC_ATTR6_REG, reg.u32);
+}
+
+static int
+sssnic_base_init(struct sssnic_hw *hw)
+{
+	int ret;
+	struct rte_pci_device *pci_dev;
+
+	PMD_INIT_FUNC_TRACE();
+
+	pci_dev = hw->pci_dev;
+
+	/* get base addresses of hw registers */
+	hw->cfg_base_addr =
+		(uint8_t *)pci_dev->mem_resource[SSSNIC_PCI_BAR_CFG].addr;
+	hw->mgmt_base_addr =
+		(uint8_t *)pci_dev->mem_resource[SSSNIC_PCI_BAR_MGMT].addr;
+	hw->db_base_addr =
+		(uint8_t *)pci_dev->mem_resource[SSSNIC_PCI_BAR_DB].addr;
+	hw->db_mem_len =
+		(uint8_t *)pci_dev->mem_resource[SSSNIC_PCI_BAR_DB].len;
+
+	ret = wait_for_sssnic_hw_ready(hw);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Hardware is not ready!");
+		return -EBUSY;
+	}
+	sssnic_attr_setup(hw);
+	ret = wait_for_sssnic_db_enabled(hw);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Doorbell is not enabled!");
+		return -EBUSY;
+	}
+	sssnic_af_setup(hw);
+	sssnic_msix_all_disable(hw);
+	sssnic_pf_status_set(hw, SSSNIC_PF_STATUS_INIT);
+
+	PMD_DRV_LOG(DEBUG,
+		"func_idx:%d, func_type:%d, pci_idx:%d, vf_off:%d, global_vf_off:%d "
+		"pf_idx:%d, af_idx:%d, mf_idx:%d, num_aeq:%d, num_ceq:%d, num_irq:%d",
+		hw->attr.func_idx, hw->attr.func_type, hw->attr.pci_idx,
+		hw->attr.vf_off, hw->attr.global_vf_off, hw->attr.pf_idx,
+		hw->attr.af_idx, hw->attr.mf_idx, hw->attr.num_aeq,
+		hw->attr.num_ceq, hw->attr.num_irq);
+
+	return 0;
+}
+
+int
+sssnic_hw_init(struct sssnic_hw *hw)
+{
+	int ret;
+
+	PMD_INIT_FUNC_TRACE();
+
+	ret = sssnic_base_init(hw);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to initialize hardware base");
+		return ret;
+	}
+
+	return -EINVAL;
+}
+
+void
+sssnic_hw_shutdown(struct sssnic_hw *hw)
+{
+	RTE_SET_USED(hw);
+	PMD_INIT_FUNC_TRACE();
+}
diff --git a/drivers/net/sssnic/base/sssnic_hw.h b/drivers/net/sssnic/base/sssnic_hw.h
index db916b1977..65d4d562b4 100644
--- a/drivers/net/sssnic/base/sssnic_hw.h
+++ b/drivers/net/sssnic/base/sssnic_hw.h
@@ -8,4 +8,53 @@
 #define SSSNIC_PCI_VENDOR_ID 0x1F3F
 #define SSSNIC_DEVICE_ID_STD 0x9020
 
+#define SSSNIC_PCI_BAR_CFG 1
+#define SSSNIC_PCI_BAR_MGMT 3
+#define SSSNIC_PCI_BAR_DB 4
+
+#define SSSNIC_FUNC_TYPE_PF 0
+#define SSSNIC_FUNC_TYPE_VF 1
+#define SSSNIC_FUNC_TYPE_AF 2
+#define SSSNIC_FUNC_TYPE_INVALID 3
+
+#define SSSNIC_DB_CTRL_ENABLE 0x0
+#define SSSNIC_DB_CTRL_DISABLE 0x1
+
+#define SSSNIC_MSIX_ENABLE 0
+#define SSSNIC_MSIX_DISABLE 1
+
+enum sssnic_pf_status {
+	SSSNIC_PF_STATUS_INIT = 0x0,
+	SSSNIC_PF_STATUS_ACTIVE = 0x11,
+	SSSNIC_PF_STATUS_START = 0x12,
+	SSSNIC_PF_STATUS_FINI = 0x13,
+};
+
+struct sssnic_hw_attr {
+	uint16_t func_idx;
+	uint8_t pf_idx;
+	uint8_t pci_idx;
+	uint8_t vf_off; /* vf offset in pf */
+	uint8_t global_vf_off;
+	uint8_t func_type;
+	uint8_t af_idx;
+	uint8_t mf_idx;
+	uint8_t num_aeq;
+	uint16_t num_ceq;
+	uint16_t num_irq;
+};
+
+struct sssnic_hw {
+	struct rte_pci_device *pci_dev;
+	uint8_t *cfg_base_addr;
+	uint8_t *mgmt_base_addr;
+	uint8_t *db_base_addr;
+	uint8_t *db_mem_len;
+	struct sssnic_hw_attr attr;
+};
+
+int sssnic_hw_init(struct sssnic_hw *hw);
+void sssnic_hw_shutdown(struct sssnic_hw *hw);
+void sssnic_msix_state_set(struct sssnic_hw *hw, uint16_t msix_id, int state);
+
 #endif /* _SSSNIC_HW_H_ */
diff --git a/drivers/net/sssnic/base/sssnic_reg.h b/drivers/net/sssnic/base/sssnic_reg.h
new file mode 100644
index 0000000000..77d83292eb
--- /dev/null
+++ b/drivers/net/sssnic/base/sssnic_reg.h
@@ -0,0 +1,169 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018-2022 Shenzhen 3SNIC Information Technology Co., Ltd.
+ */
+
+#ifndef _SSSNIC_REG_H_
+#define _SSSNIC_REG_H_
+
+#include <rte_io.h>
+
+/* registers of config */
+#define SSSNIC_ATTR0_REG 0x0
+#define SSSNIC_ATTR1_REG 0x4
+#define SSSNIC_ATTR2_REG 0x8
+#define SSSNIC_ATTR3_REG 0xC
+#define SSSNIC_ATTR4_REG 0x10
+#define SSSNIC_ATTR5_REG 0x14
+#define SSSNIC_ATTR6_REG 0x18
+
+#define SSSNIC_MSIX_CTRL_REG 0x58
+
+/* registers of mgmt */
+#define SSSNIC_AF_ELECTION_REG 0x6000
+#define SSSNIC_MF_ELECTION_REG 0x6020
+
+struct sssnic_attr0_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t func_idx : 12;
+			uint32_t pf_idx : 5;
+			uint32_t pci_idx : 3;
+			uint32_t vf_off : 8; /* vf offset in pf */
+			uint32_t func_type : 1;
+			uint32_t resvd_0 : 4;
+		};
+	};
+};
+
+struct sssnic_attr1_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t af_idx : 6;
+			uint32_t resvd_0 : 2;
+			uint32_t num_aeq : 2;
+			uint32_t resvd_1 : 20;
+			uint32_t mgmt_init_status : 1;
+			uint32_t pf_init_status : 1;
+		};
+	};
+};
+
+struct sssnic_attr2_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t num_ceq : 9;
+			uint32_t num_dma_attr : 3;
+			uint32_t resvd_0 : 4;
+			uint32_t num_irq : 11;
+			uint32_t resvd_1 : 5;
+		};
+	};
+};
+
+struct sssnic_attr3_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t global_vf_off1 : 12;
+			uint32_t resvd_0 : 4;
+			uint32_t global_vf_off : 12; /*global vf offset*/
+			uint32_t resvd_1 : 4;
+		};
+	};
+};
+
+struct sssnic_attr4_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t db_ctrl : 1;
+			uint32_t resvd_0 : 31;
+		};
+	};
+};
+
+struct sssnic_attr5_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t outbound_ctrl : 1;
+			uint32_t resvd_0 : 31;
+		};
+	};
+};
+
+struct sssnic_attr6_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t pf_status : 16;
+			uint32_t resvd_0 : 6;
+			uint32_t msix_en : 1;
+			uint32_t max_queues : 9;
+		};
+	};
+};
+
+struct sssnic_af_election_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t func_idx : 6;
+			uint32_t resvd_0 : 26;
+		};
+	};
+};
+
+struct sssnic_mf_election_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t func_idx : 5;
+			uint32_t resvd_0 : 27;
+		};
+	};
+};
+
+struct sssnic_msix_ctrl_reg {
+	union {
+		uint32_t u32;
+		struct {
+			uint32_t resend_timer_clr : 1;
+			uint32_t int_msk_set : 1;
+			uint32_t int_msk_clr : 1;
+			uint32_t auto_msk_set : 1;
+			uint32_t auto_msk_clr : 1;
+			uint32_t resvd_0 : 17;
+			uint32_t msxi_idx : 10;
+		};
+	};
+};
+
+static inline uint32_t
+sssnic_cfg_reg_read(struct sssnic_hw *hw, uint32_t reg)
+{
+	return rte_be_to_cpu_32(rte_read32(hw->cfg_base_addr + reg));
+}
+
+static inline void
+sssnic_cfg_reg_write(struct sssnic_hw *hw, uint32_t reg, uint32_t val)
+{
+	rte_write32(rte_cpu_to_be_32(val), hw->cfg_base_addr + reg);
+}
+
+static inline uint32_t
+sssnic_mgmt_reg_read(struct sssnic_hw *hw, uint32_t reg)
+{
+	return rte_be_to_cpu_32(rte_read32(hw->mgmt_base_addr + reg));
+}
+
+static inline void
+sssnic_mgmt_reg_write(struct sssnic_hw *hw, uint32_t reg, uint32_t val)
+{
+	rte_write32(rte_cpu_to_be_32(val), hw->mgmt_base_addr + reg);
+}
+
+#endif /*_SSSNIC_REG_H_*/
diff --git a/drivers/net/sssnic/meson.build b/drivers/net/sssnic/meson.build
index fda65aa380..328bb41436 100644
--- a/drivers/net/sssnic/meson.build
+++ b/drivers/net/sssnic/meson.build
@@ -13,6 +13,9 @@ if (arch_subdir != 'x86' and arch_subdir != 'arm') or (not dpdk_conf.get('RTE_AR
     subdir_done()
 endif
 
+subdir('base')
+objs = [base_objs]
+
 sources = files(
         'sssnic_ethdev.c',
 )
diff --git a/drivers/net/sssnic/sssnic_ethdev.c b/drivers/net/sssnic/sssnic_ethdev.c
index 4f8b5c2684..e198b1e1d0 100644
--- a/drivers/net/sssnic/sssnic_ethdev.c
+++ b/drivers/net/sssnic/sssnic_ethdev.c
@@ -7,25 +7,62 @@
 
 #include "sssnic_log.h"
 #include "base/sssnic_hw.h"
+#include "sssnic_ethdev.h"
+
+static void
+sssnic_ethdev_release(struct rte_eth_dev *ethdev)
+{
+	struct sssnic_hw *hw = SSSNIC_ETHDEV_TO_HW(ethdev);
+
+	sssnic_hw_shutdown(hw);
+	rte_free(hw);
+}
 
 static int
 sssnic_ethdev_init(struct rte_eth_dev *ethdev)
 {
-	RTE_SET_USED(ethdev);
+	int ret;
+	struct sssnic_hw *hw;
+	struct sssnic_netdev *netdev;
+	struct rte_pci_device *pci_dev;
+
 	PMD_INIT_FUNC_TRACE();
 
-	return -EINVAL;
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	netdev = SSSNIC_ETHDEV_PRIVATE(ethdev);
+	pci_dev = RTE_ETH_DEV_TO_PCI(ethdev);
+	hw = rte_zmalloc("sssnic_hw", sizeof(struct sssnic_hw), 0);
+	if (hw == NULL) {
+		PMD_DRV_LOG(ERR, "Failed to alloc memory for hw");
+		return -ENOMEM;
+	}
+	netdev->hw = hw;
+	hw->pci_dev = pci_dev;
+	ret = sssnic_hw_init(hw);
+	if (ret != 0) {
+		rte_free(hw);
+		return ret;
+	}
+
+	return 0;
 }
 
 static int
 sssnic_ethdev_uninit(struct rte_eth_dev *ethdev)
 {
-	RTE_SET_USED(ethdev);
 	PMD_INIT_FUNC_TRACE();
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
 
+	/* ethdev port has been released */
+	if (ethdev->state == RTE_ETH_DEV_UNUSED)
+		return 0;
+
+	sssnic_ethdev_release(ethdev);
+
 	return -EINVAL;
 }
 
@@ -35,7 +72,8 @@ sssnic_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 	RTE_SET_USED(pci_drv);
 	PMD_INIT_FUNC_TRACE();
 
-	return rte_eth_dev_pci_generic_probe(pci_dev, 0, sssnic_ethdev_init);
+	return rte_eth_dev_pci_generic_probe(pci_dev,
+		sizeof(struct sssnic_netdev), sssnic_ethdev_init);
 }
 
 static int
diff --git a/drivers/net/sssnic/sssnic_ethdev.h b/drivers/net/sssnic/sssnic_ethdev.h
new file mode 100644
index 0000000000..5d951134cc
--- /dev/null
+++ b/drivers/net/sssnic/sssnic_ethdev.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018-2022 Shenzhen 3SNIC Information Technology Co., Ltd.
+ */
+
+#ifndef _SSSNIC_ETHDEV_H_
+#define _SSSNIC_ETHDEV_H_
+
+struct sssnic_netdev {
+	void *hw;
+};
+
+#define SSSNIC_ETHDEV_PRIVATE(eth_dev)                                         \
+	((struct sssnic_netdev *)(eth_dev)->data->dev_private)
+#define SSSNIC_NETDEV_TO_HW(netdev) ((struct sssnic_hw *)(netdev)->hw)
+#define SSSNIC_ETHDEV_TO_HW(eth_dev)                                           \
+	SSSNIC_NETDEV_TO_HW(SSSNIC_ETHDEV_PRIVATE(eth_dev))
+
+#endif /*_SSSNIC_ETHDEV_H_*/
-- 
2.27.0