[PATCH v3 26/27] net/nfp: refact the cppcore and PCIe module

[Chaoyong He <chaoyong.he@corigine.com>]

Sync the logic from kernel driver, use the new entry function from the
PCIe module instead of the cppcore module.

Signed-off-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Niklas Söderlund <niklas.soderlund@corigine.com>
---
 drivers/net/nfp/nfp_ethdev.c           |   9 +-
 drivers/net/nfp/nfpcore/nfp6000_pcie.c | 197 ++++++++++---------------
 drivers/net/nfp/nfpcore/nfp6000_pcie.h |   6 +
 drivers/net/nfp/nfpcore/nfp_cpp.h      |  36 +----
 drivers/net/nfp/nfpcore/nfp_cppcore.c  |  68 ++++++---
 5 files changed, 141 insertions(+), 175 deletions(-)

diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index 5d129d0ad3..6eefec3836 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -23,6 +23,7 @@
 #include "nfpcore/nfp_mip.h"
 #include "nfpcore/nfp_rtsym.h"
 #include "nfpcore/nfp_nsp.h"
+#include "nfpcore/nfp6000_pcie.h"
 
 #include "nfp_common.h"
 #include "nfp_ctrl.h"
@@ -917,9 +918,9 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
 	 * use a lock file if UIO is being used.
 	 */
 	if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
-		cpp = nfp_cpp_from_device_name(pci_dev, 0);
+		cpp = nfp_cpp_from_nfp6000_pcie(pci_dev, false);
 	else
-		cpp = nfp_cpp_from_device_name(pci_dev, 1);
+		cpp = nfp_cpp_from_nfp6000_pcie(pci_dev, true);
 
 	if (cpp == NULL) {
 		PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
@@ -1120,9 +1121,9 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
 	 * use a lock file if UIO is being used.
 	 */
 	if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
-		cpp = nfp_cpp_from_device_name(pci_dev, 0);
+		cpp = nfp_cpp_from_nfp6000_pcie(pci_dev, false);
 	else
-		cpp = nfp_cpp_from_device_name(pci_dev, 1);
+		cpp = nfp_cpp_from_nfp6000_pcie(pci_dev, true);
 
 	if (cpp == NULL) {
 		PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
diff --git a/drivers/net/nfp/nfpcore/nfp6000_pcie.c b/drivers/net/nfp/nfpcore/nfp6000_pcie.c
index 4f453f19a9..e32ac1107c 100644
--- a/drivers/net/nfp/nfpcore/nfp6000_pcie.c
+++ b/drivers/net/nfp/nfpcore/nfp6000_pcie.c
@@ -98,6 +98,24 @@ struct nfp_pcie_user {
 	char *cfg;
 };
 
+/* Generic CPP bus access interface. */
+struct nfp6000_area_priv {
+	struct nfp_bar *bar;
+	uint32_t bar_offset;
+
+	int target;
+	int action;
+	int token;
+	uint64_t offset;
+	struct {
+		int read;
+		int write;
+		int bar;
+	} width;
+	size_t size;
+	char *iomem;
+};
+
 static uint32_t
 nfp_bar_maptype(struct nfp_bar *bar)
 {
@@ -334,24 +352,6 @@ nfp_disable_bars(struct nfp_pcie_user *nfp)
 	}
 }
 
-/* Generic CPP bus access interface. */
-struct nfp6000_area_priv {
-	struct nfp_bar *bar;
-	uint32_t bar_offset;
-
-	uint32_t target;
-	uint32_t action;
-	uint32_t token;
-	uint64_t offset;
-	struct {
-		int read;
-		int write;
-		int bar;
-	} width;
-	size_t size;
-	char *iomem;
-};
-
 static int
 nfp6000_area_init(struct nfp_cpp_area *area,
 		uint32_t dest,
@@ -624,87 +624,6 @@ nfp_acquire_process_lock(struct nfp_pcie_user *desc)
 	return 0;
 }
 
-static int
-nfp6000_set_model(struct rte_pci_device *dev,
-		struct nfp_cpp *cpp)
-{
-	uint32_t model;
-
-	if (rte_pci_read_config(dev, &model, 4, 0x2e) < 0) {
-		PMD_DRV_LOG(ERR, "nfp set model failed");
-		return -1;
-	}
-
-	model  = model << 16;
-	nfp_cpp_model_set(cpp, model);
-
-	return 0;
-}
-
-static int
-nfp6000_set_interface(struct rte_pci_device *dev,
-		struct nfp_cpp *cpp)
-{
-	uint16_t interface;
-
-	if (rte_pci_read_config(dev, &interface, 2, 0x154) < 0) {
-		PMD_DRV_LOG(ERR, "nfp set interface failed");
-		return -1;
-	}
-
-	nfp_cpp_interface_set(cpp, interface);
-
-	return 0;
-}
-
-static int
-nfp6000_set_serial(struct rte_pci_device *dev,
-		struct nfp_cpp *cpp)
-{
-	off_t pos;
-	uint16_t tmp;
-	uint8_t serial[6];
-	int serial_len = 6;
-
-	pos = rte_pci_find_ext_capability(dev, RTE_PCI_EXT_CAP_ID_DSN);
-	if (pos <= 0) {
-		PMD_DRV_LOG(ERR, "PCI_EXT_CAP_ID_DSN not found. nfp set serial failed");
-		return -1;
-	} else {
-		pos += 6;
-	}
-
-	if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
-		PMD_DRV_LOG(ERR, "nfp set serial failed");
-		return -1;
-	}
-
-	serial[4] = (uint8_t)((tmp >> 8) & 0xff);
-	serial[5] = (uint8_t)(tmp & 0xff);
-
-	pos += 2;
-	if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
-		PMD_DRV_LOG(ERR, "nfp set serial failed");
-		return -1;
-	}
-
-	serial[2] = (uint8_t)((tmp >> 8) & 0xff);
-	serial[3] = (uint8_t)(tmp & 0xff);
-
-	pos += 2;
-	if (rte_pci_read_config(dev, &tmp, 2, pos) < 0) {
-		PMD_DRV_LOG(ERR, "nfp set serial failed");
-		return -1;
-	}
-
-	serial[0] = (uint8_t)((tmp >> 8) & 0xff);
-	serial[1] = (uint8_t)(tmp & 0xff);
-
-	nfp_cpp_serial_set(cpp, serial, serial_len);
-
-	return 0;
-}
-
 static int
 nfp6000_get_dsn(struct rte_pci_device *pci_dev,
 		uint64_t *dsn)
@@ -795,12 +714,7 @@ nfp6000_init(struct nfp_cpp *cpp,
 		struct rte_pci_device *dev)
 {
 	int ret = 0;
-	struct nfp_pcie_user *desc;
-
-	desc = malloc(sizeof(*desc));
-	if (desc == NULL)
-		return -1;
-
+	struct nfp_pcie_user *desc = nfp_cpp_priv(cpp);
 
 	memset(desc->busdev, 0, BUSDEV_SZ);
 	strlcpy(desc->busdev, dev->device.name, sizeof(desc->busdev));
@@ -809,17 +723,11 @@ nfp6000_init(struct nfp_cpp *cpp,
 			nfp_cpp_driver_need_lock(cpp)) {
 		ret = nfp_acquire_process_lock(desc);
 		if (ret != 0)
-			goto error;
+			return -1;
 	}
 
-	if (nfp6000_set_model(dev, cpp) < 0)
-		goto error;
-	if (nfp6000_set_interface(dev, cpp) < 0)
-		goto error;
-	if (nfp6000_set_serial(dev, cpp) < 0)
-		goto error;
 	if (nfp6000_set_barsz(dev, desc) < 0)
-		goto error;
+		return -1;
 
 	desc->cfg = dev->mem_resource[0].addr;
 	desc->dev_id = dev->addr.function & 0x7;
@@ -830,13 +738,7 @@ nfp6000_init(struct nfp_cpp *cpp,
 		return -1;
 	}
 
-	nfp_cpp_priv_set(cpp, desc);
-
 	return 0;
-
-error:
-	free(desc);
-	return -1;
 }
 
 static void
@@ -848,7 +750,7 @@ nfp6000_free(struct nfp_cpp *cpp)
 	if (nfp_cpp_driver_need_lock(cpp))
 		close(desc->lock);
 	close(desc->device);
-	free(desc);
+	rte_free(desc);
 }
 
 static const struct nfp_cpp_operations nfp6000_pcie_ops = {
@@ -873,3 +775,58 @@ nfp_cpp_operations *nfp_cpp_transport_operations(void)
 {
 	return &nfp6000_pcie_ops;
 }
+
+/**
+ * Build a NFP CPP bus from a NFP6000 PCI device
+ *
+ * @param pdev
+ *   NFP6000 PCI device
+ * @param driver_lock_needed
+ *   driver lock flag
+ *
+ * @return
+ *   NFP CPP handle or NULL
+ */
+struct nfp_cpp *
+nfp_cpp_from_nfp6000_pcie(struct rte_pci_device *pci_dev,
+		bool driver_lock_needed)
+{
+	int ret;
+	struct nfp_cpp *cpp;
+	uint16_t interface = 0;
+	struct nfp_pcie_user *nfp;
+
+	nfp = rte_zmalloc(NULL, sizeof(*nfp), 0);
+	if (nfp == NULL)
+		return NULL;
+
+	ret = nfp6000_get_interface(pci_dev, &interface);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Get interface failed.");
+		rte_free(nfp);
+		return NULL;
+	}
+
+	if (NFP_CPP_INTERFACE_TYPE_of(interface) != NFP_CPP_INTERFACE_TYPE_PCI) {
+		PMD_DRV_LOG(ERR, "Interface type is not right.");
+		rte_free(nfp);
+		return NULL;
+	}
+
+	if (NFP_CPP_INTERFACE_CHANNEL_of(interface) !=
+			NFP_CPP_INTERFACE_CHANNEL_PEROPENER) {
+		PMD_DRV_LOG(ERR, "Interface channel is not right");
+		rte_free(nfp);
+		return NULL;
+	}
+
+	/* Probe for all the common NFP devices */
+	cpp = nfp_cpp_from_device_name(pci_dev, nfp, driver_lock_needed);
+	if (cpp == NULL) {
+		PMD_DRV_LOG(ERR, "Get cpp from operation failed");
+		rte_free(nfp);
+		return NULL;
+	}
+
+	return cpp;
+}
diff --git a/drivers/net/nfp/nfpcore/nfp6000_pcie.h b/drivers/net/nfp/nfpcore/nfp6000_pcie.h
index e126457d45..8847f6f946 100644
--- a/drivers/net/nfp/nfpcore/nfp6000_pcie.h
+++ b/drivers/net/nfp/nfpcore/nfp6000_pcie.h
@@ -6,8 +6,14 @@
 #ifndef __NFP6000_PCIE_H__
 #define __NFP6000_PCIE_H__
 
+#include <ethdev_pci.h>
+
 #include "nfp_cpp.h"
+#include "nfp_dev.h"
 
 const struct nfp_cpp_operations *nfp_cpp_transport_operations(void);
 
+struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct rte_pci_device *pci_dev,
+		bool driver_lock_needed);
+
 #endif /* __NFP6000_PCIE_H__ */
diff --git a/drivers/net/nfp/nfpcore/nfp_cpp.h b/drivers/net/nfp/nfpcore/nfp_cpp.h
index 0f36ba0b50..e879c7c920 100644
--- a/drivers/net/nfp/nfpcore/nfp_cpp.h
+++ b/drivers/net/nfp/nfpcore/nfp_cpp.h
@@ -226,40 +226,12 @@ uint32_t nfp_cpp_model_autodetect(struct nfp_cpp *cpp, uint32_t *model);
 
 /* NFP CPP core interface for CPP clients */
 struct nfp_cpp *nfp_cpp_from_device_name(struct rte_pci_device *dev,
-		int driver_lock_needed);
+		void *priv, bool driver_lock_needed);
 
 void nfp_cpp_free(struct nfp_cpp *cpp);
 
 #define NFP_CPP_MODEL_INVALID   0xffffffff
 
-/**
- * Retrieve the chip ID from the model ID
- *
- * The chip ID is a 16-bit BCD+A-F encoding for the chip type.
- *
- * @param model
- *   NFP CPP model id
- *
- * @return
- *   NFP CPP chip id
- */
-#define NFP_CPP_MODEL_CHIP_of(model)        (((model) >> 16) & 0xffff)
-
-/**
- * Check for the NFP6000 family of devices
- *
- * NOTE: The NFP4000 series is considered as a NFP6000 series variant.
- *
- * @param model
- *   NFP CPP model id
- *
- * @return
- *   true if model is in the NFP6000 family, false otherwise.
- */
-#define NFP_CPP_MODEL_IS_6000(model)                         \
-		((NFP_CPP_MODEL_CHIP_of(model) >= 0x3800) && \
-		(NFP_CPP_MODEL_CHIP_of(model) < 0x7000))
-
 uint32_t nfp_cpp_model(struct nfp_cpp *cpp);
 
 /*
@@ -330,6 +302,12 @@ uint32_t nfp_cpp_model(struct nfp_cpp *cpp);
  */
 #define NFP_CPP_INTERFACE_CHANNEL_of(interface)	(((interface) >>  0) & 0xff)
 
+/*
+ * Use this channel ID for multiple virtual channel interfaces
+ * (ie ARM and PCIe) when setting up the interface field.
+ */
+#define NFP_CPP_INTERFACE_CHANNEL_PEROPENER    255
+
 uint16_t nfp_cpp_interface(struct nfp_cpp *cpp);
 
 uint32_t nfp_cpp_serial(struct nfp_cpp *cpp, const uint8_t **serial);
diff --git a/drivers/net/nfp/nfpcore/nfp_cppcore.c b/drivers/net/nfp/nfpcore/nfp_cppcore.c
index 0c1a03b0ab..d70c8cd44a 100644
--- a/drivers/net/nfp/nfpcore/nfp_cppcore.c
+++ b/drivers/net/nfp/nfpcore/nfp_cppcore.c
@@ -909,10 +909,13 @@ nfp_xpb_readl(struct nfp_cpp *cpp,
 }
 
 static struct nfp_cpp *
-nfp_cpp_alloc(struct rte_pci_device *dev,
-		int driver_lock_needed)
+nfp_cpp_alloc(struct rte_pci_device *pci_dev,
+		void *priv,
+		bool driver_lock_needed)
 {
 	int err;
+	size_t target;
+	uint32_t xpb_addr;
 	struct nfp_cpp *cpp;
 	const struct nfp_cpp_operations *ops;
 
@@ -925,32 +928,50 @@ nfp_cpp_alloc(struct rte_pci_device *dev,
 		return NULL;
 
 	cpp->op = ops;
+	cpp->priv = priv;
 	cpp->driver_lock_needed = driver_lock_needed;
 
-	if (cpp->op->init) {
-		err = cpp->op->init(cpp, dev);
+	err = ops->get_interface(pci_dev, &cpp->interface);
+	if (err != 0) {
+		rte_free(cpp);
+		return NULL;
+	}
+
+	err = ops->get_serial(pci_dev, cpp->serial, NFP_SERIAL_LEN);
+	if (err != 0) {
+		rte_free(cpp);
+		return NULL;
+	}
+
+	/*
+	 * NOTE: cpp_lock is NOT locked for op->init,
+	 * since it may call NFP CPP API operations
+	 */
+	err = cpp->op->init(cpp, pci_dev);
+	if (err < 0) {
+		PMD_DRV_LOG(ERR, "NFP interface initialization failed");
+		rte_free(cpp);
+		return NULL;
+	}
+
+	err = nfp_cpp_model_autodetect(cpp, &cpp->model);
+	if (err < 0) {
+		PMD_DRV_LOG(ERR, "NFP model detection failed");
+		rte_free(cpp);
+		return NULL;
+	}
+
+	for (target = 0; target < RTE_DIM(cpp->imb_cat_table); target++) {
+		/* Hardcoded XPB IMB Base, island 0 */
+		xpb_addr = 0x000a0000 + (target * 4);
+		err = nfp_xpb_readl(cpp, xpb_addr, &cpp->imb_cat_table[target]);
 		if (err < 0) {
+			PMD_DRV_LOG(ERR, "Can't read CPP mapping from device");
 			rte_free(cpp);
 			return NULL;
 		}
 	}
 
-	if (NFP_CPP_MODEL_IS_6000(nfp_cpp_model(cpp))) {
-		uint32_t xpb_addr;
-		size_t target;
-
-		for (target = 0; target < RTE_DIM(cpp->imb_cat_table); target++) {
-			/* Hardcoded XPB IMB Base, island 0 */
-			xpb_addr = 0x000a0000 + (target * 4);
-			err = nfp_xpb_readl(cpp, xpb_addr,
-					(uint32_t *)&cpp->imb_cat_table[target]);
-			if (err < 0) {
-				rte_free(cpp);
-				return NULL;
-			}
-		}
-	}
-
 	err = nfp_cpp_set_mu_locality_lsb(cpp);
 	if (err < 0) {
 		PMD_DRV_LOG(ERR, "Can't calculate MU locality bit offset");
@@ -981,6 +1002,8 @@ nfp_cpp_free(struct nfp_cpp *cpp)
  *
  * @param dev
  *   PCI device
+ * @param priv
+ *   Private data of low-level implementation
  * @param driver_lock_needed
  *   Driver lock flag
  *
@@ -991,9 +1014,10 @@ nfp_cpp_free(struct nfp_cpp *cpp)
  */
 struct nfp_cpp *
 nfp_cpp_from_device_name(struct rte_pci_device *dev,
-		int driver_lock_needed)
+		void *priv,
+		bool driver_lock_needed)
 {
-	return nfp_cpp_alloc(dev, driver_lock_needed);
+	return nfp_cpp_alloc(dev, priv, driver_lock_needed);
 }
 
 /**
-- 
2.39.1