* [PATCH 01/11] net/nfp: refactor the probe logic of the secondary process
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 02/11] net/nfp: fix the failure to initialize the LSC mask Chaoyong He
` (10 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Chaoyong He, Peng Zhang, Long Wu
The probe logic of the secondary process of PF PMD now is not very
similarly with the logic of the primary process, which cause we need two
different logics when we add new feature in some case.
Refactor the probe logic of the secondary process to solve this problem.
Signed-off-by: Chaoyong He <chaoyong.he@corigine.com>
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/net/nfp/flower/nfp_flower.c | 4 +--
drivers/net/nfp/flower/nfp_flower.h | 2 +-
drivers/net/nfp/nfp_ethdev.c | 42 ++++++++++++++++++++++-------
3 files changed, 35 insertions(+), 13 deletions(-)
diff --git a/drivers/net/nfp/flower/nfp_flower.c b/drivers/net/nfp/flower/nfp_flower.c
index f2e6eb6a6f..6b523d98b0 100644
--- a/drivers/net/nfp/flower/nfp_flower.c
+++ b/drivers/net/nfp/flower/nfp_flower.c
@@ -859,7 +859,7 @@ nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev,
}
int
-nfp_secondary_init_app_fw_flower(struct nfp_cpp *cpp)
+nfp_secondary_init_app_fw_flower(struct nfp_pf_dev *pf_dev)
{
struct rte_eth_dev *eth_dev;
const char *port_name = "pf_vnic_eth_dev";
@@ -872,7 +872,7 @@ nfp_secondary_init_app_fw_flower(struct nfp_cpp *cpp)
return -ENODEV;
}
- eth_dev->process_private = cpp;
+ eth_dev->process_private = pf_dev->cpp;
eth_dev->dev_ops = &nfp_flower_pf_vnic_ops;
eth_dev->rx_pkt_burst = nfp_net_recv_pkts;
eth_dev->tx_pkt_burst = nfp_flower_pf_xmit_pkts;
diff --git a/drivers/net/nfp/flower/nfp_flower.h b/drivers/net/nfp/flower/nfp_flower.h
index 220b714018..6f27c06acc 100644
--- a/drivers/net/nfp/flower/nfp_flower.h
+++ b/drivers/net/nfp/flower/nfp_flower.h
@@ -106,7 +106,7 @@ nfp_flower_support_decap_v2(const struct nfp_app_fw_flower *app_fw_flower)
int nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev,
const struct nfp_dev_info *dev_info);
-int nfp_secondary_init_app_fw_flower(struct nfp_cpp *cpp);
+int nfp_secondary_init_app_fw_flower(struct nfp_pf_dev *pf_dev);
bool nfp_flower_pf_dispatch_pkts(struct nfp_net_hw *hw,
struct rte_mbuf *mbuf,
uint32_t port_id);
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index 4fae2e5540..705465046c 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -1006,9 +1006,7 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
}
static int
-nfp_secondary_init_app_fw_nic(struct rte_pci_device *pci_dev,
- struct nfp_rtsym_table *sym_tbl,
- struct nfp_cpp *cpp)
+nfp_secondary_init_app_fw_nic(struct nfp_pf_dev *pf_dev)
{
uint32_t i;
int err = 0;
@@ -1017,7 +1015,7 @@ nfp_secondary_init_app_fw_nic(struct rte_pci_device *pci_dev,
struct nfp_net_hw *hw;
/* Read the number of vNIC's created for the PF */
- total_vnics = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
+ total_vnics = nfp_rtsym_read_le(pf_dev->sym_tbl, "nfd_cfg_pf0_num_ports", &err);
if (err != 0 || total_vnics == 0 || total_vnics > 8) {
PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value");
return -ENODEV;
@@ -1027,7 +1025,7 @@ nfp_secondary_init_app_fw_nic(struct rte_pci_device *pci_dev,
struct rte_eth_dev *eth_dev;
char port_name[RTE_ETH_NAME_MAX_LEN];
snprintf(port_name, sizeof(port_name), "%s_port%u",
- pci_dev->device.name, i);
+ pf_dev->pci_dev->device.name, i);
PMD_INIT_LOG(DEBUG, "Secondary attaching to port %s", port_name);
eth_dev = rte_eth_dev_attach_secondary(port_name);
@@ -1037,7 +1035,7 @@ nfp_secondary_init_app_fw_nic(struct rte_pci_device *pci_dev,
break;
}
- eth_dev->process_private = cpp;
+ eth_dev->process_private = pf_dev->cpp;
hw = eth_dev->data->dev_private;
nfp_net_ethdev_ops_mount(hw, eth_dev);
@@ -1057,7 +1055,9 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
{
int ret = 0;
struct nfp_cpp *cpp;
+ struct nfp_pf_dev *pf_dev;
enum nfp_app_fw_id app_fw_id;
+ char name[RTE_ETH_NAME_MAX_LEN];
struct nfp_rtsym_table *sym_tbl;
const struct nfp_dev_info *dev_info;
@@ -1075,6 +1075,14 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
return -ENODEV;
}
+ /* Allocate memory for the PF "device" */
+ snprintf(name, sizeof(name), "nfp_pf%d", 0);
+ pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
+ if (pf_dev == NULL) {
+ PMD_INIT_LOG(ERR, "Can't allocate memory for the PF device");
+ return -ENOMEM;
+ }
+
/*
* When device bound to UIO, the device could be used, by mistake,
* by two DPDK apps, and the UIO driver does not avoid it. This
@@ -1089,7 +1097,8 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
if (cpp == NULL) {
PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
- return -EIO;
+ ret = -EIO;
+ goto pf_cleanup;
}
/*
@@ -1099,20 +1108,29 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
sym_tbl = nfp_rtsym_table_read(cpp);
if (sym_tbl == NULL) {
PMD_INIT_LOG(ERR, "Something is wrong with the firmware symbol table");
- return -EIO;
+ ret = -EIO;
+ goto pf_cleanup;
}
/* Read the app ID of the firmware loaded */
app_fw_id = nfp_rtsym_read_le(sym_tbl, "_pf0_net_app_id", &ret);
if (ret != 0) {
PMD_INIT_LOG(ERR, "Couldn't read app_fw_id from fw");
+ ret = -EIO;
goto sym_tbl_cleanup;
}
+ /* Populate the newly created PF device */
+ pf_dev->app_fw_id = app_fw_id;
+ pf_dev->cpp = cpp;
+ pf_dev->sym_tbl = sym_tbl;
+ pf_dev->pci_dev = pci_dev;
+
+ /* Call app specific init code now */
switch (app_fw_id) {
case NFP_APP_FW_CORE_NIC:
PMD_INIT_LOG(INFO, "Initializing coreNIC");
- ret = nfp_secondary_init_app_fw_nic(pci_dev, sym_tbl, cpp);
+ ret = nfp_secondary_init_app_fw_nic(pf_dev);
if (ret != 0) {
PMD_INIT_LOG(ERR, "Could not initialize coreNIC!");
goto sym_tbl_cleanup;
@@ -1120,7 +1138,7 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
break;
case NFP_APP_FW_FLOWER_NIC:
PMD_INIT_LOG(INFO, "Initializing Flower");
- ret = nfp_secondary_init_app_fw_flower(cpp);
+ ret = nfp_secondary_init_app_fw_flower(pf_dev);
if (ret != 0) {
PMD_INIT_LOG(ERR, "Could not initialize Flower!");
goto sym_tbl_cleanup;
@@ -1132,8 +1150,12 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
goto sym_tbl_cleanup;
}
+ return 0;
+
sym_tbl_cleanup:
free(sym_tbl);
+pf_cleanup:
+ rte_free(pf_dev);
return ret;
}
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 02/11] net/nfp: fix the failure to initialize the LSC mask
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
2023-11-02 2:23 ` [PATCH 01/11] net/nfp: refactor the probe logic of the secondary process Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 03/11] net/nfp: fix the DMA error caused by app exit abnormally Chaoyong He
` (9 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, stable, Shihong Wang, Chaoyong He
From: Peng Zhang <peng.zhang@corigine.com>
In rare cases, when DPDK application exit, the interrupt handler was not
processed the interrupt in time, resulting in the LSC interrupt mask bit
not being cleared. So when the DPDK application start again, the newly
coming LSC interrupts cannot be received and processed properly.
Fix this problem by force clear the LSC interrupt mask on port
initialization.
Fixes: 6c53f87b3497 ("nfp: add link status interrupt")
Cc: stable@dpdk.org
Signed-off-by: Shihong Wang <shihong.wang@corigine.com>
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
---
drivers/net/nfp/nfp_ethdev.c | 2 ++
drivers/net/nfp/nfp_ethdev_vf.c | 2 ++
drivers/net/nfp/nfp_net_common.c | 2 +-
drivers/net/nfp/nfp_net_common.h | 1 +
4 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index 705465046c..abaf31e27b 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -590,6 +590,8 @@ nfp_net_init(struct rte_eth_dev *eth_dev)
nfp_net_dev_interrupt_handler, (void *)eth_dev);
/* Telling the firmware about the LSC interrupt entry */
nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
+ /* Unmask the LSC interrupt */
+ nfp_net_irq_unmask(eth_dev);
/* Recording current stats counters values */
nfp_net_stats_reset(eth_dev);
diff --git a/drivers/net/nfp/nfp_ethdev_vf.c b/drivers/net/nfp/nfp_ethdev_vf.c
index f3aa649054..cc345e9218 100644
--- a/drivers/net/nfp/nfp_ethdev_vf.c
+++ b/drivers/net/nfp/nfp_ethdev_vf.c
@@ -351,6 +351,8 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev)
nfp_net_dev_interrupt_handler, (void *)eth_dev);
/* Telling the firmware about the LSC interrupt entry */
nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
+ /* Unmask the LSC interrupt */
+ nfp_net_irq_unmask(eth_dev);
/* Recording current stats counters values */
nfp_net_stats_reset(eth_dev);
diff --git a/drivers/net/nfp/nfp_net_common.c b/drivers/net/nfp/nfp_net_common.c
index 4efcdff76f..f8ef049a42 100644
--- a/drivers/net/nfp/nfp_net_common.c
+++ b/drivers/net/nfp/nfp_net_common.c
@@ -1319,7 +1319,7 @@ nfp_net_dev_link_status_print(struct rte_eth_dev *dev)
* If MSI-X auto-masking is enabled clear the mask bit, otherwise
* clear the ICR for the entry.
*/
-static void
+void
nfp_net_irq_unmask(struct rte_eth_dev *dev)
{
struct nfp_net_hw *hw;
diff --git a/drivers/net/nfp/nfp_net_common.h b/drivers/net/nfp/nfp_net_common.h
index 1f9001c81d..b9df2fe563 100644
--- a/drivers/net/nfp/nfp_net_common.h
+++ b/drivers/net/nfp/nfp_net_common.h
@@ -205,6 +205,7 @@ int nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
int nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
void nfp_net_params_setup(struct nfp_net_hw *hw);
void nfp_net_cfg_queue_setup(struct nfp_net_hw *hw);
+void nfp_net_irq_unmask(struct rte_eth_dev *dev);
void nfp_net_dev_interrupt_handler(void *param);
void nfp_net_dev_interrupt_delayed_handler(void *param);
int nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 03/11] net/nfp: fix the DMA error caused by app exit abnormally
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
2023-11-02 2:23 ` [PATCH 01/11] net/nfp: refactor the probe logic of the secondary process Chaoyong He
2023-11-02 2:23 ` [PATCH 02/11] net/nfp: fix the failure to initialize the LSC mask Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 04/11] net/nfp: add flag to indicate multiple PFs support Chaoyong He
` (8 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Shihong Wang, stable, Peng Zhang, Chaoyong He
From: Shihong Wang <shihong.wang@corigine.com>
When DPDK application exit abnormally, there might have DMA error,
and which will cause the load of firmware failed.
Fix this by force the physical port down to clear the possible DMA error.
Fixes: 896c265ef954 ("net/nfp: use new CPP interface")
Cc: stable@dpdk.org
Signed-off-by: Shihong Wang <shihong.wang@corigine.com>
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
---
drivers/net/nfp/nfp_ethdev.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index abaf31e27b..aa2b59af32 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -847,6 +847,7 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
static int
nfp_pf_init(struct rte_pci_device *pci_dev)
{
+ uint32_t i;
int ret = 0;
uint64_t addr;
uint32_t cpp_id;
@@ -905,6 +906,10 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
goto hwinfo_cleanup;
}
+ /* Force the physical port down to clear the possible DMA error */
+ for (i = 0; i < nfp_eth_table->count; i++)
+ nfp_eth_set_configured(cpp, nfp_eth_table->ports[i].index, 0);
+
if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo) != 0) {
PMD_INIT_LOG(ERR, "Error when uploading firmware");
ret = -EIO;
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 04/11] net/nfp: add flag to indicate multiple PFs support
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (2 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 03/11] net/nfp: fix the DMA error caused by app exit abnormally Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 05/11] net/nfp: add major version to nsp commands Chaoyong He
` (7 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, Chaoyong He, Long Wu
From: Peng Zhang <peng.zhang@corigine.com>
Support for multiple PFs have been added to the NFP3800 firmware. This
can be detected by reading the NSP major version, which was bumped to 1
when support was added.
Add a flag and detecting method to record if the current device is
cabable to support multiple PFs. This will be used in later patches to
initialize and make use of this new feature.
Noteworthy about the detection method from NSP version information, the
NSP minor version was not touched when increasing the major version.
This makes the first NSP version to support multiple PFs version 1.8,
while the latest version without this supports remains 0.8.
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/net/nfp/nfp_ethdev.c | 49 +++++++++++++++++++++++--------
drivers/net/nfp/nfp_net_common.h | 8 +++++
drivers/net/nfp/nfpcore/nfp_nsp.c | 14 +++++++--
3 files changed, 56 insertions(+), 15 deletions(-)
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index aa2b59af32..7022ef435f 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -479,7 +479,7 @@ nfp_net_init(struct rte_eth_dev *eth_dev)
rte_eth_copy_pci_info(eth_dev, pci_dev);
- if (port == 0) {
+ if (port == 0 || pf_dev->multi_pf.enabled) {
uint32_t min_size;
hw->ctrl_bar = pf_dev->ctrl_bar;
@@ -712,6 +712,26 @@ nfp_fw_setup(struct rte_pci_device *dev,
return err;
}
+static inline bool
+nfp_check_multi_pf_from_nsp(struct rte_pci_device *pci_dev,
+ struct nfp_cpp *cpp)
+{
+ bool flag;
+ struct nfp_nsp *nsp;
+
+ nsp = nfp_nsp_open(cpp);
+ if (nsp == NULL) {
+ PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle");
+ return false;
+ }
+
+ flag = (nfp_nsp_get_abi_ver_major(nsp) > 0) &&
+ (pci_dev->id.device_id == PCI_DEVICE_ID_NFP3800_PF_NIC);
+
+ nfp_nsp_close(nsp);
+ return flag;
+}
+
static int
nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
const struct nfp_dev_info *dev_info)
@@ -874,6 +894,14 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
return -ENODEV;
}
+ /* Allocate memory for the PF "device" */
+ snprintf(name, sizeof(name), "nfp_pf%d", 0);
+ pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
+ if (pf_dev == NULL) {
+ PMD_INIT_LOG(ERR, "Can't allocate memory for the PF device");
+ return -ENOMEM;
+ }
+
/*
* When device bound to UIO, the device could be used, by mistake,
* by two DPDK apps, and the UIO driver does not avoid it. This
@@ -888,7 +916,8 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
if (cpp == NULL) {
PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
- return -EIO;
+ ret = -EIO;
+ goto pf_cleanup;
}
hwinfo = nfp_hwinfo_read(cpp);
@@ -906,6 +935,8 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
goto hwinfo_cleanup;
}
+ pf_dev->multi_pf.enabled = nfp_check_multi_pf_from_nsp(pci_dev, cpp);
+
/* Force the physical port down to clear the possible DMA error */
for (i = 0; i < nfp_eth_table->count; i++)
nfp_eth_set_configured(cpp, nfp_eth_table->ports[i].index, 0);
@@ -932,14 +963,6 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
goto sym_tbl_cleanup;
}
- /* Allocate memory for the PF "device" */
- snprintf(name, sizeof(name), "nfp_pf%d", 0);
- pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
- if (pf_dev == NULL) {
- ret = -ENOMEM;
- goto sym_tbl_cleanup;
- }
-
/* Populate the newly created PF device */
pf_dev->app_fw_id = app_fw_id;
pf_dev->cpp = cpp;
@@ -957,7 +980,7 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
if (pf_dev->qc_bar == NULL) {
PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc");
ret = -EIO;
- goto pf_cleanup;
+ goto sym_tbl_cleanup;
}
PMD_INIT_LOG(DEBUG, "qc_bar address: %p", pf_dev->qc_bar);
@@ -998,8 +1021,6 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
hwqueues_cleanup:
nfp_cpp_area_free(pf_dev->qc_area);
-pf_cleanup:
- rte_free(pf_dev);
sym_tbl_cleanup:
free(sym_tbl);
eth_table_cleanup:
@@ -1008,6 +1029,8 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
free(hwinfo);
cpp_cleanup:
nfp_cpp_free(cpp);
+pf_cleanup:
+ rte_free(pf_dev);
return ret;
}
diff --git a/drivers/net/nfp/nfp_net_common.h b/drivers/net/nfp/nfp_net_common.h
index b9df2fe563..bd0ed077c5 100644
--- a/drivers/net/nfp/nfp_net_common.h
+++ b/drivers/net/nfp/nfp_net_common.h
@@ -54,6 +54,11 @@ struct nfp_net_tlv_caps {
uint32_t mbox_cmsg_types; /**< Cmsgs which can be passed through the mailbox */
};
+struct nfp_multi_pf {
+ /** Support multiple PF */
+ bool enabled;
+};
+
struct nfp_pf_dev {
/** Backpointer to associated pci device */
struct rte_pci_device *pci_dev;
@@ -79,6 +84,9 @@ struct nfp_pf_dev {
/** Service id of cpp bridge service */
uint32_t cpp_bridge_id;
+
+ /** Multiple PF configuration */
+ struct nfp_multi_pf multi_pf;
};
struct nfp_app_fw_nic {
diff --git a/drivers/net/nfp/nfpcore/nfp_nsp.c b/drivers/net/nfp/nfpcore/nfp_nsp.c
index a680b972b8..9f88b822f3 100644
--- a/drivers/net/nfp/nfpcore/nfp_nsp.c
+++ b/drivers/net/nfp/nfpcore/nfp_nsp.c
@@ -40,7 +40,17 @@
#define NSP_DFLT_BUFFER_SIZE_MB GENMASK_ULL(7, 0)
#define NSP_MAGIC 0xab10
-#define NSP_MAJOR 0
+
+/*
+ * ABI major version is bumped separately without resetting minor
+ * version when the change in NSP is not compatible to old driver.
+ */
+#define NSP_MAJOR 1
+
+/*
+ * ABI minor version is bumped when new feature is introduced
+ * while old driver can still work without this new feature.
+ */
#define NSP_MINOR 8
#define NSP_CODE_MAJOR GENMASK_ULL(15, 12)
@@ -203,7 +213,7 @@ nfp_nsp_check(struct nfp_nsp *state)
state->ver.major = FIELD_GET(NSP_STATUS_MAJOR, reg);
state->ver.minor = FIELD_GET(NSP_STATUS_MINOR, reg);
- if (state->ver.major != NSP_MAJOR || state->ver.minor < NSP_MINOR) {
+ if (state->ver.major > NSP_MAJOR || state->ver.minor < NSP_MINOR) {
PMD_DRV_LOG(ERR, "Unsupported ABI %hu.%hu", state->ver.major,
state->ver.minor);
return -EINVAL;
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 05/11] net/nfp: add major version to nsp commands
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (3 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 04/11] net/nfp: add flag to indicate multiple PFs support Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 06/11] net/nfp: adjust physical port check for multiple PFs Chaoyong He
` (6 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, Chaoyong He, Long Wu
From: Peng Zhang <peng.zhang@corigine.com>
The commands sent to the NSP take the NSP major version into account. Up
until now only NSP major version 0 have been supported and the value
have been hard-coded to 0.
In preparation to add support for both NSP version 0.x and 1.x, extend
the command to take the running NSP version into account.
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/net/nfp/nfpcore/nfp_nsp.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/nfp/nfpcore/nfp_nsp.c b/drivers/net/nfp/nfpcore/nfp_nsp.c
index 9f88b822f3..589d878e0d 100644
--- a/drivers/net/nfp/nfpcore/nfp_nsp.c
+++ b/drivers/net/nfp/nfpcore/nfp_nsp.c
@@ -22,7 +22,8 @@
#define NSP_COMMAND 0x08
#define NSP_COMMAND_OPTION GENMASK_ULL(63, 32)
-#define NSP_COMMAND_CODE GENMASK_ULL(31, 16)
+#define NSP_COMMAND_VER_MAJOR GENMASK_ULL(31, 28)
+#define NSP_COMMAND_CODE GENMASK_ULL(27, 16)
#define NSP_COMMAND_DMA_BUF RTE_BIT64(1)
#define NSP_COMMAND_START RTE_BIT64(0)
@@ -370,6 +371,7 @@ nfp_nsp_command_real(struct nfp_nsp *state,
err = nfp_cpp_writeq(cpp, nsp_cpp, nsp_command,
FIELD_PREP(NSP_COMMAND_OPTION, arg->option) |
+ FIELD_PREP(NSP_COMMAND_VER_MAJOR, state->ver.major) |
FIELD_PREP(NSP_COMMAND_CODE, arg->code) |
FIELD_PREP(NSP_COMMAND_DMA_BUF, arg->dma) |
FIELD_PREP(NSP_COMMAND_START, 1));
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 06/11] net/nfp: adjust physical port check for multiple PFs
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (4 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 05/11] net/nfp: add major version to nsp commands Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 07/11] net/nfp: add the check about the firmware load Chaoyong He
` (5 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, Chaoyong He, Long Wu
From: Peng Zhang <peng.zhang@corigine.com>
If the firmware supports multiple PFs each PF is represented by a single
physical port. While if the firmware only supports a single PF there
might be one or more physical ports represented by a single PF.
Adjust the check to handle both single and multiple PFs firmware.
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/net/nfp/nfp_ethdev.c | 33 +++++++++++++++++++++++++--------
1 file changed, 25 insertions(+), 8 deletions(-)
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index 7022ef435f..3ebfd444b3 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -712,6 +712,15 @@ nfp_fw_setup(struct rte_pci_device *dev,
return err;
}
+static inline bool
+nfp_check_multi_pf_from_fw(uint32_t total_vnics)
+{
+ if (total_vnics == 1)
+ return true;
+
+ return false;
+}
+
static inline bool
nfp_check_multi_pf_from_nsp(struct rte_pci_device *pci_dev,
struct nfp_cpp *cpp)
@@ -765,14 +774,22 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
goto app_cleanup;
}
- /*
- * For coreNIC the number of vNICs exposed should be the same as the
- * number of physical ports.
- */
- if (total_vnics != nfp_eth_table->count) {
- PMD_INIT_LOG(ERR, "Total physical ports do not match number of vNICs");
- ret = -ENODEV;
- goto app_cleanup;
+ if (pf_dev->multi_pf.enabled) {
+ if (!nfp_check_multi_pf_from_fw(total_vnics)) {
+ PMD_INIT_LOG(ERR, "NSP report multipf, but FW report not multipf");
+ ret = -ENODEV;
+ goto app_cleanup;
+ }
+ } else {
+ /*
+ * For coreNIC the number of vNICs exposed should be the same as the
+ * number of physical ports.
+ */
+ if (total_vnics != nfp_eth_table->count) {
+ PMD_INIT_LOG(ERR, "Total physical ports do not match number of vNICs");
+ ret = -ENODEV;
+ goto app_cleanup;
+ }
}
/* Populate coreNIC app properties */
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 07/11] net/nfp: add the check about the firmware load
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (5 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 06/11] net/nfp: adjust physical port check for multiple PFs Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 08/11] net/nfp: add PF ID used to format symbols Chaoyong He
` (4 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, Chaoyong He, Long Wu
From: Peng Zhang <peng.zhang@corigine.com>
When firmware load failed, it doesn't have any notice.
So add the check about the firmware load and add an exit
point when the firmware load process fail.
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/net/nfp/nfp_ethdev.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index 3ebfd444b3..9378a2ebc3 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -654,7 +654,12 @@ nfp_fw_upload(struct rte_pci_device *dev,
PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
fw_name, fsize);
PMD_DRV_LOG(INFO, "Uploading the firmware ...");
- nfp_nsp_load_fw(nsp, fw_buf, fsize);
+ if (nfp_nsp_load_fw(nsp, fw_buf, fsize) < 0) {
+ free(fw_buf);
+ PMD_DRV_LOG(ERR, "Firmware load failed.");
+ return -EIO;
+ }
+
PMD_DRV_LOG(INFO, "Done");
free(fw_buf);
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 08/11] net/nfp: add PF ID used to format symbols
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (6 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 07/11] net/nfp: add the check about the firmware load Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 09/11] net/nfp: add nsp command to check if firmware is loaded Chaoyong He
` (3 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, Chaoyong He, Long Wu
From: Peng Zhang <peng.zhang@corigine.com>
In single PF scenario, the format symbols just is related
with PF ID 0. In multiple PF scenario, the format symbols
should be related with PF ID. So this commit adds the
PF ID used to format symbols.
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/net/nfp/nfp_ethdev.c | 109 ++++++++++++++++++++++++-------
drivers/net/nfp/nfp_net_common.h | 2 +
2 files changed, 86 insertions(+), 25 deletions(-)
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index 9378a2ebc3..96f0ae3fe3 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -224,11 +224,22 @@ nfp_net_set_link_down(struct rte_eth_dev *dev)
return nfp_eth_set_configured(dev->process_private, hw->nfp_idx, 0);
}
+static uint8_t
+nfp_function_id_get(const struct nfp_pf_dev *pf_dev,
+ uint8_t phy_port)
+{
+ if (pf_dev->multi_pf.enabled)
+ return pf_dev->multi_pf.function_id;
+
+ return phy_port;
+}
+
/* Reset and stop device. The device can not be restarted. */
static int
nfp_net_close(struct rte_eth_dev *dev)
{
uint8_t i;
+ uint8_t id;
struct nfp_net_hw *hw;
struct nfp_pf_dev *pf_dev;
struct rte_pci_device *pci_dev;
@@ -264,8 +275,10 @@ nfp_net_close(struct rte_eth_dev *dev)
app_fw_nic->ports[hw->idx] = NULL;
for (i = 0; i < app_fw_nic->total_phyports; i++) {
+ id = nfp_function_id_get(pf_dev, i);
+
/* Check to see if ports are still in use */
- if (app_fw_nic->ports[i] != NULL)
+ if (app_fw_nic->ports[id] != NULL)
return 0;
}
@@ -667,6 +680,19 @@ nfp_fw_upload(struct rte_pci_device *dev,
return 0;
}
+static void
+nfp_fw_unload(struct nfp_cpp *cpp)
+{
+ struct nfp_nsp *nsp;
+
+ nsp = nfp_nsp_open(cpp);
+ if (nsp == NULL)
+ return;
+
+ nfp_nsp_device_soft_reset(nsp);
+ nfp_nsp_close(nsp);
+}
+
static int
nfp_fw_setup(struct rte_pci_device *dev,
struct nfp_cpp *cpp,
@@ -751,6 +777,7 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
const struct nfp_dev_info *dev_info)
{
uint8_t i;
+ uint8_t id;
int ret = 0;
uint32_t total_vnics;
struct nfp_net_hw *hw;
@@ -758,10 +785,13 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
struct rte_eth_dev *eth_dev;
struct nfp_app_fw_nic *app_fw_nic;
struct nfp_eth_table *nfp_eth_table;
+ char bar_name[RTE_ETH_NAME_MAX_LEN];
char port_name[RTE_ETH_NAME_MAX_LEN];
+ char vnic_name[RTE_ETH_NAME_MAX_LEN];
nfp_eth_table = pf_dev->nfp_eth_table;
PMD_INIT_LOG(INFO, "Total physical ports: %d", nfp_eth_table->count);
+ id = nfp_function_id_get(pf_dev, 0);
/* Allocate memory for the CoreNIC app */
app_fw_nic = rte_zmalloc("nfp_app_fw_nic", sizeof(*app_fw_nic), 0);
@@ -772,9 +802,10 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
pf_dev->app_fw_priv = app_fw_nic;
/* Read the number of vNIC's created for the PF */
- total_vnics = nfp_rtsym_read_le(pf_dev->sym_tbl, "nfd_cfg_pf0_num_ports", &ret);
+ snprintf(vnic_name, sizeof(vnic_name), "nfd_cfg_pf%u_num_ports", id);
+ total_vnics = nfp_rtsym_read_le(pf_dev->sym_tbl, vnic_name, &ret);
if (ret != 0 || total_vnics == 0 || total_vnics > 8) {
- PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value");
+ PMD_INIT_LOG(ERR, "%s symbol with wrong value", vnic_name);
ret = -ENODEV;
goto app_cleanup;
}
@@ -804,11 +835,12 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
app_fw_nic->multiport = true;
/* Map the symbol table */
- pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_bar0",
+ snprintf(bar_name, sizeof(bar_name), "_pf%u_net_bar0", id);
+ pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, bar_name,
app_fw_nic->total_phyports * NFP_NET_CFG_BAR_SZ,
&pf_dev->ctrl_area);
if (pf_dev->ctrl_bar == NULL) {
- PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _pf0_net_ctrl_bar");
+ PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for %s", bar_name);
ret = -EIO;
goto app_cleanup;
}
@@ -818,8 +850,9 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
/* Loop through all physical ports on PF */
numa_node = rte_socket_id();
for (i = 0; i < app_fw_nic->total_phyports; i++) {
- snprintf(port_name, sizeof(port_name), "%s_port%d",
- pf_dev->pci_dev->device.name, i);
+ id = nfp_function_id_get(pf_dev, i);
+ snprintf(port_name, sizeof(port_name), "%s_port%u",
+ pf_dev->pci_dev->device.name, id);
/* Allocate a eth_dev for this phyport */
eth_dev = rte_eth_dev_allocate(port_name);
@@ -841,14 +874,14 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
hw = eth_dev->data->dev_private;
/* Add this device to the PF's array of physical ports */
- app_fw_nic->ports[i] = hw;
+ app_fw_nic->ports[id] = hw;
hw->dev_info = dev_info;
hw->pf_dev = pf_dev;
hw->cpp = pf_dev->cpp;
hw->eth_dev = eth_dev;
- hw->idx = i;
- hw->nfp_idx = nfp_eth_table->ports[i].index;
+ hw->idx = id;
+ hw->nfp_idx = nfp_eth_table->ports[id].index;
eth_dev->device = &pf_dev->pci_dev->device;
@@ -870,13 +903,15 @@ nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
port_cleanup:
for (i = 0; i < app_fw_nic->total_phyports; i++) {
- if (app_fw_nic->ports[i] != NULL &&
- app_fw_nic->ports[i]->eth_dev != NULL) {
+ id = nfp_function_id_get(pf_dev, i);
+
+ if (app_fw_nic->ports[id] != NULL &&
+ app_fw_nic->ports[id]->eth_dev != NULL) {
struct rte_eth_dev *tmp_dev;
- tmp_dev = app_fw_nic->ports[i]->eth_dev;
+ tmp_dev = app_fw_nic->ports[id]->eth_dev;
nfp_ipsec_uninit(tmp_dev);
rte_eth_dev_release_port(tmp_dev);
- app_fw_nic->ports[i] = NULL;
+ app_fw_nic->ports[id] = NULL;
}
}
nfp_cpp_area_free(pf_dev->ctrl_area);
@@ -890,15 +925,19 @@ static int
nfp_pf_init(struct rte_pci_device *pci_dev)
{
uint32_t i;
+ uint32_t id;
int ret = 0;
uint64_t addr;
+ uint32_t index;
uint32_t cpp_id;
+ uint8_t function_id;
struct nfp_cpp *cpp;
struct nfp_pf_dev *pf_dev;
struct nfp_hwinfo *hwinfo;
enum nfp_app_fw_id app_fw_id;
char name[RTE_ETH_NAME_MAX_LEN];
struct nfp_rtsym_table *sym_tbl;
+ char app_name[RTE_ETH_NAME_MAX_LEN];
struct nfp_eth_table *nfp_eth_table;
const struct nfp_dev_info *dev_info;
@@ -917,7 +956,8 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
}
/* Allocate memory for the PF "device" */
- snprintf(name, sizeof(name), "nfp_pf%d", 0);
+ function_id = (pci_dev->addr.function) & 0x07;
+ snprintf(name, sizeof(name), "nfp_pf%u", function_id);
pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
if (pf_dev == NULL) {
PMD_INIT_LOG(ERR, "Can't allocate memory for the PF device");
@@ -958,10 +998,14 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
}
pf_dev->multi_pf.enabled = nfp_check_multi_pf_from_nsp(pci_dev, cpp);
+ pf_dev->multi_pf.function_id = function_id;
/* Force the physical port down to clear the possible DMA error */
- for (i = 0; i < nfp_eth_table->count; i++)
- nfp_eth_set_configured(cpp, nfp_eth_table->ports[i].index, 0);
+ for (i = 0; i < nfp_eth_table->count; i++) {
+ id = nfp_function_id_get(pf_dev, i);
+ index = nfp_eth_table->ports[id].index;
+ nfp_eth_set_configured(cpp, index, 0);
+ }
if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo) != 0) {
PMD_INIT_LOG(ERR, "Error when uploading firmware");
@@ -974,13 +1018,14 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
if (sym_tbl == NULL) {
PMD_INIT_LOG(ERR, "Something is wrong with the firmware symbol table");
ret = -EIO;
- goto eth_table_cleanup;
+ goto fw_cleanup;
}
/* Read the app ID of the firmware loaded */
- app_fw_id = nfp_rtsym_read_le(sym_tbl, "_pf0_net_app_id", &ret);
+ snprintf(app_name, sizeof(app_name), "_pf%u_net_app_id", function_id);
+ app_fw_id = nfp_rtsym_read_le(sym_tbl, app_name, &ret);
if (ret != 0) {
- PMD_INIT_LOG(ERR, "Couldn't read app_fw_id from fw");
+ PMD_INIT_LOG(ERR, "Couldn't read %s from firmware", app_name);
ret = -EIO;
goto sym_tbl_cleanup;
}
@@ -1045,6 +1090,8 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
nfp_cpp_area_free(pf_dev->qc_area);
sym_tbl_cleanup:
free(sym_tbl);
+fw_cleanup:
+ nfp_fw_unload(cpp);
eth_table_cleanup:
free(nfp_eth_table);
hwinfo_cleanup:
@@ -1063,21 +1110,29 @@ nfp_secondary_init_app_fw_nic(struct nfp_pf_dev *pf_dev)
uint32_t i;
int err = 0;
int ret = 0;
+ uint8_t function_id;
uint32_t total_vnics;
struct nfp_net_hw *hw;
+ char pf_name[RTE_ETH_NAME_MAX_LEN];
/* Read the number of vNIC's created for the PF */
- total_vnics = nfp_rtsym_read_le(pf_dev->sym_tbl, "nfd_cfg_pf0_num_ports", &err);
+ function_id = (pf_dev->pci_dev->addr.function) & 0x07;
+ snprintf(pf_name, sizeof(pf_name), "nfd_cfg_pf%u_num_ports", function_id);
+ total_vnics = nfp_rtsym_read_le(pf_dev->sym_tbl, pf_name, &err);
if (err != 0 || total_vnics == 0 || total_vnics > 8) {
- PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value");
+ PMD_INIT_LOG(ERR, "%s symbol with wrong value", pf_name);
return -ENODEV;
}
for (i = 0; i < total_vnics; i++) {
+ uint32_t id = i;
struct rte_eth_dev *eth_dev;
char port_name[RTE_ETH_NAME_MAX_LEN];
+
+ if (nfp_check_multi_pf_from_fw(total_vnics))
+ id = function_id;
snprintf(port_name, sizeof(port_name), "%s_port%u",
- pf_dev->pci_dev->device.name, i);
+ pf_dev->pci_dev->device.name, id);
PMD_INIT_LOG(DEBUG, "Secondary attaching to port %s", port_name);
eth_dev = rte_eth_dev_attach_secondary(port_name);
@@ -1107,11 +1162,13 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
{
int ret = 0;
struct nfp_cpp *cpp;
+ uint8_t function_id;
struct nfp_pf_dev *pf_dev;
enum nfp_app_fw_id app_fw_id;
char name[RTE_ETH_NAME_MAX_LEN];
struct nfp_rtsym_table *sym_tbl;
const struct nfp_dev_info *dev_info;
+ char app_name[RTE_ETH_NAME_MAX_LEN];
if (pci_dev == NULL)
return -ENODEV;
@@ -1165,9 +1222,11 @@ nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
}
/* Read the app ID of the firmware loaded */
- app_fw_id = nfp_rtsym_read_le(sym_tbl, "_pf0_net_app_id", &ret);
+ function_id = pci_dev->addr.function & 0x7;
+ snprintf(app_name, sizeof(app_name), "_pf%u_net_app_id", function_id);
+ app_fw_id = nfp_rtsym_read_le(sym_tbl, app_name, &ret);
if (ret != 0) {
- PMD_INIT_LOG(ERR, "Couldn't read app_fw_id from fw");
+ PMD_INIT_LOG(ERR, "Couldn't read %s from fw", app_name);
ret = -EIO;
goto sym_tbl_cleanup;
}
diff --git a/drivers/net/nfp/nfp_net_common.h b/drivers/net/nfp/nfp_net_common.h
index bd0ed077c5..a40ddfd985 100644
--- a/drivers/net/nfp/nfp_net_common.h
+++ b/drivers/net/nfp/nfp_net_common.h
@@ -57,6 +57,8 @@ struct nfp_net_tlv_caps {
struct nfp_multi_pf {
/** Support multiple PF */
bool enabled;
+ /** Function index */
+ uint8_t function_id;
};
struct nfp_pf_dev {
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 09/11] net/nfp: add nsp command to check if firmware is loaded
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (7 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 08/11] net/nfp: add PF ID used to format symbols Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 10/11] net/nfp: introduce keepalive mechanism for multiple PF Chaoyong He
` (2 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, Chaoyong He, Long Wu
From: Peng Zhang <peng.zhang@corigine.com>
Add a NSP command to check if any firmware have been loaded.
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/net/nfp/nfpcore/nfp_nsp.c | 6 ++++++
drivers/net/nfp/nfpcore/nfp_nsp.h | 1 +
2 files changed, 7 insertions(+)
diff --git a/drivers/net/nfp/nfpcore/nfp_nsp.c b/drivers/net/nfp/nfpcore/nfp_nsp.c
index 589d878e0d..e5aaef8d55 100644
--- a/drivers/net/nfp/nfpcore/nfp_nsp.c
+++ b/drivers/net/nfp/nfpcore/nfp_nsp.c
@@ -637,6 +637,12 @@ nfp_nsp_load_fw(struct nfp_nsp *state,
return 0;
}
+bool
+nfp_nsp_fw_loaded(struct nfp_nsp *state)
+{
+ return nfp_nsp_command(state, SPCODE_FW_LOADED) > 0;
+}
+
int
nfp_nsp_read_eth_table(struct nfp_nsp *state,
void *buf,
diff --git a/drivers/net/nfp/nfpcore/nfp_nsp.h b/drivers/net/nfp/nfpcore/nfp_nsp.h
index fe52dffeb7..492fa7e99f 100644
--- a/drivers/net/nfp/nfpcore/nfp_nsp.h
+++ b/drivers/net/nfp/nfpcore/nfp_nsp.h
@@ -212,5 +212,6 @@ enum nfp_nsp_sensor_id {
int nfp_hwmon_read_sensor(struct nfp_cpp *cpp, enum nfp_nsp_sensor_id id,
uint32_t *val);
+bool nfp_nsp_fw_loaded(struct nfp_nsp *state);
#endif /* __NSP_NSP_H__ */
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 10/11] net/nfp: introduce keepalive mechanism for multiple PF
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (8 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 09/11] net/nfp: add nsp command to check if firmware is loaded Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 2:23 ` [PATCH 11/11] drivers: enable multiple PF in application firmware Chaoyong He
2023-11-02 14:52 ` [PATCH 00/11] Add the support of multiple PF Ferruh Yigit
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, Chaoyong He, Long Wu
From: Peng Zhang <peng.zhang@corigine.com>
In multiple PF scenario, management firmware is in charge of
application firmware unloading instead of driver by keepalive
mechanism.
A new NSP resource area is allocated for keepalive use with name
"nfp.beat". Driver sets periodically updates the PFs' corresponding
word in "nfp.beat". Management firmware checks these PF's words to
learn whether and which PF are alive, and will unload the application
firmware if no PF is running.
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/net/nfp/nfp_ethdev.c | 173 ++++++++++++++++++++++++-
drivers/net/nfp/nfp_net_common.h | 17 +++
drivers/net/nfp/nfpcore/nfp_resource.h | 3 +
3 files changed, 189 insertions(+), 4 deletions(-)
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index 96f0ae3fe3..bbc0109f5f 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -5,6 +5,8 @@
* Small portions derived from code Copyright(c) 2010-2015 Intel Corporation.
*/
+#include <unistd.h>
+
#include <eal_firmware.h>
#include <rte_alarm.h>
@@ -16,6 +18,7 @@
#include "nfpcore/nfp_rtsym.h"
#include "nfpcore/nfp_nsp.h"
#include "nfpcore/nfp6000_pcie.h"
+#include "nfpcore/nfp_resource.h"
#include "nfp_cpp_bridge.h"
#include "nfp_ipsec.h"
@@ -234,6 +237,79 @@ nfp_function_id_get(const struct nfp_pf_dev *pf_dev,
return phy_port;
}
+static void
+nfp_net_beat_timer(void *arg)
+{
+ uint64_t cur_sec;
+ struct nfp_multi_pf *multi_pf = arg;
+
+ cur_sec = rte_rdtsc();
+ nn_writeq(cur_sec, multi_pf->beat_addr + NFP_BEAT_OFFSET(multi_pf->function_id));
+
+ /* Beat once per second. */
+ if (rte_eal_alarm_set(1000 * 1000, nfp_net_beat_timer,
+ (void *)multi_pf) < 0) {
+ PMD_DRV_LOG(ERR, "Error setting alarm");
+ }
+}
+
+static int
+nfp_net_keepalive_init(struct nfp_cpp *cpp,
+ struct nfp_multi_pf *multi_pf)
+{
+ uint8_t *base;
+ uint64_t addr;
+ uint32_t size;
+ uint32_t cpp_id;
+ struct nfp_resource *res;
+
+ res = nfp_resource_acquire(cpp, NFP_RESOURCE_KEEPALIVE);
+ if (res == NULL)
+ return -EIO;
+
+ cpp_id = nfp_resource_cpp_id(res);
+ addr = nfp_resource_address(res);
+ size = nfp_resource_size(res);
+
+ nfp_resource_release(res);
+
+ /* Allocate a fixed area for keepalive. */
+ base = nfp_cpp_map_area(cpp, cpp_id, addr, size, &multi_pf->beat_area);
+ if (base == NULL) {
+ PMD_DRV_LOG(ERR, "Failed to map area for keepalive.");
+ return -EIO;
+ }
+
+ multi_pf->beat_addr = base;
+
+ return 0;
+}
+
+static void
+nfp_net_keepalive_uninit(struct nfp_multi_pf *multi_pf)
+{
+ nfp_cpp_area_release_free(multi_pf->beat_area);
+}
+
+static int
+nfp_net_keepalive_start(struct nfp_multi_pf *multi_pf)
+{
+ if (rte_eal_alarm_set(1000 * 1000, nfp_net_beat_timer,
+ (void *)multi_pf) < 0) {
+ PMD_DRV_LOG(ERR, "Error setting alarm");
+ return -EIO;
+ }
+
+ return 0;
+}
+
+static void
+nfp_net_keepalive_stop(struct nfp_multi_pf *multi_pf)
+{
+ /* Cancel keepalive for multiple PF setup */
+ rte_eal_alarm_cancel(nfp_net_beat_timer, (void *)multi_pf);
+}
+
/* Reset and stop device. The device can not be restarted. */
static int
nfp_net_close(struct rte_eth_dev *dev)
@@ -284,6 +360,10 @@ nfp_net_close(struct rte_eth_dev *dev)
/* Now it is safe to free all PF resources */
PMD_INIT_LOG(INFO, "Freeing PF resources");
+ if (pf_dev->multi_pf.enabled) {
+ nfp_net_keepalive_stop(&pf_dev->multi_pf);
+ nfp_net_keepalive_uninit(&pf_dev->multi_pf);
+ }
nfp_cpp_area_free(pf_dev->ctrl_area);
nfp_cpp_area_free(pf_dev->qc_area);
free(pf_dev->hwinfo);
@@ -693,11 +773,92 @@ nfp_fw_unload(struct nfp_cpp *cpp)
nfp_nsp_close(nsp);
}
+static int
+nfp_fw_reload(struct rte_pci_device *dev,
+ struct nfp_nsp *nsp,
+ char *card_desc)
+{
+ int err;
+
+ nfp_nsp_device_soft_reset(nsp);
+ err = nfp_fw_upload(dev, nsp, card_desc);
+ if (err != 0)
+ PMD_DRV_LOG(ERR, "NFP firmware load failed");
+
+ return err;
+}
+
+static int
+nfp_fw_loaded_check_alive(struct rte_pci_device *dev,
+ struct nfp_nsp *nsp,
+ char *card_desc,
+ const struct nfp_dev_info *dev_info,
+ struct nfp_multi_pf *multi_pf)
+{
+ int offset;
+ uint32_t i;
+ uint64_t beat;
+ uint32_t port_num;
+
+ /*
+ * If the beats of any other port changed in 3s,
+ * we should not reload the firmware.
+ */
+ for (port_num = 0; port_num < dev_info->pf_num_per_unit; port_num++) {
+ if (port_num == multi_pf->function_id)
+ continue;
+
+ offset = NFP_BEAT_OFFSET(port_num);
+ beat = nn_readq(multi_pf->beat_addr + offset);
+ for (i = 0; i < 3; i++) {
+ sleep(1);
+ if (nn_readq(multi_pf->beat_addr + offset) != beat)
+ return 0;
+ }
+ }
+
+ return nfp_fw_reload(dev, nsp, card_desc);
+}
+
+static int
+nfp_fw_reload_for_multipf(struct rte_pci_device *dev,
+ struct nfp_nsp *nsp,
+ char *card_desc,
+ struct nfp_cpp *cpp,
+ const struct nfp_dev_info *dev_info,
+ struct nfp_multi_pf *multi_pf)
+{
+ int err;
+
+ err = nfp_net_keepalive_init(cpp, multi_pf);
+ if (err != 0)
+ PMD_DRV_LOG(ERR, "NFP write beat failed");
+
+ if (nfp_nsp_fw_loaded(nsp))
+ err = nfp_fw_loaded_check_alive(dev, nsp, card_desc, dev_info, multi_pf);
+ else
+ err = nfp_fw_reload(dev, nsp, card_desc);
+ if (err != 0) {
+ nfp_net_keepalive_uninit(multi_pf);
+ return err;
+ }
+
+ err = nfp_net_keepalive_start(multi_pf);
+ if (err != 0) {
+ nfp_net_keepalive_uninit(multi_pf);
+ PMD_DRV_LOG(ERR, "NFP write beat failed");
+ }
+
+ return err;
+}
+
static int
nfp_fw_setup(struct rte_pci_device *dev,
struct nfp_cpp *cpp,
struct nfp_eth_table *nfp_eth_table,
- struct nfp_hwinfo *hwinfo)
+ struct nfp_hwinfo *hwinfo,
+ const struct nfp_dev_info *dev_info,
+ struct nfp_multi_pf *multi_pf)
{
int err;
char card_desc[100];
@@ -736,8 +897,10 @@ nfp_fw_setup(struct rte_pci_device *dev,
return -EIO;
}
- nfp_nsp_device_soft_reset(nsp);
- err = nfp_fw_upload(dev, nsp, card_desc);
+ if (multi_pf->enabled)
+ err = nfp_fw_reload_for_multipf(dev, nsp, card_desc, cpp, dev_info, multi_pf);
+ else
+ err = nfp_fw_reload(dev, nsp, card_desc);
nfp_nsp_close(nsp);
return err;
@@ -1007,7 +1170,8 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
nfp_eth_set_configured(cpp, index, 0);
}
- if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo) != 0) {
+ if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo,
+ dev_info, &pf_dev->multi_pf) != 0) {
PMD_INIT_LOG(ERR, "Error when uploading firmware");
ret = -EIO;
goto eth_table_cleanup;
@@ -1092,6 +1256,7 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
free(sym_tbl);
fw_cleanup:
nfp_fw_unload(cpp);
+ nfp_net_keepalive_stop(&pf_dev->multi_pf);
eth_table_cleanup:
free(nfp_eth_table);
hwinfo_cleanup:
diff --git a/drivers/net/nfp/nfp_net_common.h b/drivers/net/nfp/nfp_net_common.h
index a40ddfd985..db2e5842ca 100644
--- a/drivers/net/nfp/nfp_net_common.h
+++ b/drivers/net/nfp/nfp_net_common.h
@@ -36,6 +36,19 @@
/* Number of supported physical ports */
#define NFP_MAX_PHYPORTS 12
+#define NFP_BEAT_LENGTH 8
+
+/*
+ * Each PF has corresponding word to beat:
+ * Offset | Usage
+ * 0 | magic number
+ * 8 | beat of Pf0
+ * 16 | beat of Pf1
+ * 24 | beat of Pf2
+ * 32 | beat of Pf3
+ */
+#define NFP_BEAT_OFFSET(_x) (((_x) + 1) * NFP_BEAT_LENGTH)
+
/* Firmware application ID's */
enum nfp_app_fw_id {
NFP_APP_FW_CORE_NIC = 0x1,
@@ -59,6 +72,10 @@ struct nfp_multi_pf {
bool enabled;
/** Function index */
uint8_t function_id;
+ /** Pointer to CPP area for beat to keepalive */
+ struct nfp_cpp_area *beat_area;
+ /** Pointer to mapped beat address used for keepalive */
+ uint8_t *beat_addr;
};
struct nfp_pf_dev {
diff --git a/drivers/net/nfp/nfpcore/nfp_resource.h b/drivers/net/nfp/nfpcore/nfp_resource.h
index f49c99e462..7ce4727b37 100644
--- a/drivers/net/nfp/nfpcore/nfp_resource.h
+++ b/drivers/net/nfp/nfpcore/nfp_resource.h
@@ -17,6 +17,9 @@
/* Service Processor */
#define NFP_RESOURCE_NSP "nfp.sp"
+/* Keepalive */
+#define NFP_RESOURCE_KEEPALIVE "nfp.beat"
+
/* Opaque handle to a NFP Resource */
struct nfp_resource;
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH 11/11] drivers: enable multiple PF in application firmware
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (9 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 10/11] net/nfp: introduce keepalive mechanism for multiple PF Chaoyong He
@ 2023-11-02 2:23 ` Chaoyong He
2023-11-02 14:52 ` [PATCH 00/11] Add the support of multiple PF Ferruh Yigit
11 siblings, 0 replies; 13+ messages in thread
From: Chaoyong He @ 2023-11-02 2:23 UTC (permalink / raw)
To: dev; +Cc: oss-drivers, Peng Zhang, Chaoyong He, Long Wu
From: Peng Zhang <peng.zhang@corigine.com>
For backward compatibility concern, the new application firmware
is designed to support both single PF scenario and multiple PF scenario.
Thus driver should inform application firmware which setup current
is. This should be done as early as possible since the setup may
affect some configurations exposed by firmware.
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
---
drivers/common/nfp/nfp_common_ctrl.h | 1 +
drivers/net/nfp/nfp_ethdev.c | 50 ++++++++++++++++++++++++++++
2 files changed, 51 insertions(+)
diff --git a/drivers/common/nfp/nfp_common_ctrl.h b/drivers/common/nfp/nfp_common_ctrl.h
index 7033c8ea00..8a8a2c5efc 100644
--- a/drivers/common/nfp/nfp_common_ctrl.h
+++ b/drivers/common/nfp/nfp_common_ctrl.h
@@ -221,6 +221,7 @@ struct nfp_net_fw_ver {
#define NFP_NET_CFG_CTRL_IPSEC (0x1 << 1) /**< IPsec offload */
#define NFP_NET_CFG_CTRL_IPSEC_SM_LOOKUP (0x1 << 3) /**< SA short match lookup */
#define NFP_NET_CFG_CTRL_IPSEC_LM_LOOKUP (0x1 << 4) /**< SA long match lookup */
+#define NFP_NET_CFG_CTRL_MULTI_PF (0x1 << 5)
#define NFP_NET_CFG_CTRL_IN_ORDER (0x1 << 11) /**< Virtio in-order flag */
#define NFP_NET_CFG_CAP_WORD1 0x00a4
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index bbc0109f5f..e5f1d9f6f1 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -935,6 +935,50 @@ nfp_check_multi_pf_from_nsp(struct rte_pci_device *pci_dev,
return flag;
}
+static int
+nfp_enable_multi_pf(struct nfp_pf_dev *pf_dev)
+{
+ int err = 0;
+ uint64_t tx_base;
+ uint8_t *ctrl_bar;
+ struct nfp_hw *hw;
+ uint32_t cap_extend;
+ struct nfp_net_hw net_hw;
+ struct nfp_cpp_area *area;
+ char name[RTE_ETH_NAME_MAX_LEN];
+
+ memset(&net_hw, 0, sizeof(struct nfp_net_hw));
+
+ /* Map the symbol table */
+ snprintf(name, sizeof(name), "_pf%u_net_bar0",
+ pf_dev->multi_pf.function_id);
+ ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, name, NFP_NET_CFG_BAR_SZ,
+ &area);
+ if (ctrl_bar == NULL) {
+ PMD_INIT_LOG(ERR, "Failed to find data vNIC memory symbol");
+ return -ENODEV;
+ }
+
+ hw = &net_hw.super;
+ hw->ctrl_bar = ctrl_bar;
+
+ cap_extend = nn_cfg_readl(hw, NFP_NET_CFG_CAP_WORD1);
+ if ((cap_extend & NFP_NET_CFG_CTRL_MULTI_PF) == 0) {
+ PMD_INIT_LOG(ERR, "Loaded firmware doesn't support multiple PF");
+ err = -EINVAL;
+ goto end;
+ }
+
+ tx_base = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ);
+ net_hw.tx_bar = pf_dev->qc_bar + tx_base * NFP_QCP_QUEUE_ADDR_SZ;
+ nfp_net_cfg_queue_setup(&net_hw);
+ rte_spinlock_init(&hw->reconfig_lock);
+ nfp_ext_reconfig(&net_hw.super, NFP_NET_CFG_CTRL_MULTI_PF, NFP_NET_CFG_UPDATE_GEN);
+end:
+ nfp_cpp_area_release_free(area);
+ return err;
+}
+
static int
nfp_init_app_fw_nic(struct nfp_pf_dev *pf_dev,
const struct nfp_dev_info *dev_info)
@@ -1222,6 +1266,12 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
*/
switch (pf_dev->app_fw_id) {
case NFP_APP_FW_CORE_NIC:
+ if (pf_dev->multi_pf.enabled) {
+ ret = nfp_enable_multi_pf(pf_dev);
+ if (ret != 0)
+ goto hwqueues_cleanup;
+ }
+
PMD_INIT_LOG(INFO, "Initializing coreNIC");
ret = nfp_init_app_fw_nic(pf_dev, dev_info);
if (ret != 0) {
--
2.39.1
^ permalink raw reply [flat|nested] 13+ messages in thread* Re: [PATCH 00/11] Add the support of multiple PF
2023-11-02 2:23 [PATCH 00/11] Add the support of multiple PF Chaoyong He
` (10 preceding siblings ...)
2023-11-02 2:23 ` [PATCH 11/11] drivers: enable multiple PF in application firmware Chaoyong He
@ 2023-11-02 14:52 ` Ferruh Yigit
11 siblings, 0 replies; 13+ messages in thread
From: Ferruh Yigit @ 2023-11-02 14:52 UTC (permalink / raw)
To: Chaoyong He, dev; +Cc: oss-drivers
On 11/2/2023 2:23 AM, Chaoyong He wrote:
> Up to now, the NFP card using only one PF (or BDF) for multiple physical
> ports, this force the PMD import the difference logic for 'PF' and
> 'physical port'. Which is not easy to understand and also not compatible
> with some DPDK applications.
> This patch series add the support of multiple PF, which will remove this
> complexity by make sure one 'PF' for one 'physical port' with the help of
> firmware.
>
> Chaoyong He (1):
> net/nfp: refactor the probe logic of the secondary process
>
> Peng Zhang (9):
> net/nfp: fix the failure to initialize the LSC mask
> net/nfp: add flag to indicate multiple PFs support
> net/nfp: add major version to nsp commands
> net/nfp: adjust physical port check for multiple PFs
> net/nfp: add the check about the firmware load
> net/nfp: add PF ID used to format symbols
> net/nfp: add nsp command to check if firmware is loaded
> net/nfp: introduce keepalive mechanism for multiple PF
> drivers: enable multiple PF in application firmware
>
> Shihong Wang (1):
> net/nfp: fix the DMA error caused by app exit abnormally
>
Series applied to dpdk-next-net/main, thanks.
^ permalink raw reply [flat|nested] 13+ messages in thread