[dpdk-dev] [PATCH 4/4] net/hns3: restores bus_master_en and msix_enable during PF FLR reset

["Wei Hu (Xavier)" <xavier.huwei@tom.com>]

From: Chunsong Feng <fengchunsong@huawei.com>

PF FLR resets the PCIe ECAM space of all VFs under the PF and does not
automatically recover. Therefore, the VF driver needs to restore the ECAM
configuration, including bus_master_en, msix_enable.

Signed-off-by: Chunsong Feng <fengchunsong@huawei.com>
Signed-off-by: Wei Hu (Xavier) <xavier.huwei@huawei.com>
---
 drivers/net/hns3/hns3_cmd.c       |  15 ----
 drivers/net/hns3/hns3_ethdev.c    |   5 ++
 drivers/net/hns3/hns3_ethdev_vf.c | 135 +++++++++++++++++++++++++++++-
 drivers/net/hns3/hns3_intr.c      |   5 +-
 4 files changed, 139 insertions(+), 21 deletions(-)

diff --git a/drivers/net/hns3/hns3_cmd.c b/drivers/net/hns3/hns3_cmd.c
index 58776c2ec..65a5af8e4 100644
--- a/drivers/net/hns3/hns3_cmd.c
+++ b/drivers/net/hns3/hns3_cmd.c
@@ -216,24 +216,9 @@ hns3_cmd_csq_clean(struct hns3_hw *hw)
 
 	if (!is_valid_csq_clean_head(csq, head)) {
 		struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw);
-		uint32_t global;
-		uint32_t fun_rst;
 		hns3_err(hw, "wrong cmd head (%u, %u-%u)", head,
 			    csq->next_to_use, csq->next_to_clean);
 		rte_atomic16_set(&hw->reset.disable_cmd, 1);
-		if (hns->is_vf) {
-			global = hns3_read_dev(hw, HNS3_VF_RST_ING);
-			fun_rst = hns3_read_dev(hw, HNS3_FUN_RST_ING);
-			hns3_err(hw, "Delayed VF reset global: %x fun_rst: %x",
-				 global, fun_rst);
-			hns3_atomic_set_bit(HNS3_VF_RESET, &hw->reset.pending);
-		} else {
-			global = hns3_read_dev(hw, HNS3_GLOBAL_RESET_REG);
-			fun_rst = hns3_read_dev(hw, HNS3_FUN_RST_ING);
-			hns3_err(hw, "Delayed IMP reset global: %x fun_rst: %x",
-				 global, fun_rst);
-			hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending);
-		}
 
 		hns3_schedule_delayed_reset(hns);
 
diff --git a/drivers/net/hns3/hns3_ethdev.c b/drivers/net/hns3/hns3_ethdev.c
index 862a717fd..3435bce26 100644
--- a/drivers/net/hns3/hns3_ethdev.c
+++ b/drivers/net/hns3/hns3_ethdev.c
@@ -4680,6 +4680,11 @@ hns3_reset_service(void *param)
 		rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_REQUESTED);
 		hns3_err(hw, "Handling interrupts in delayed tasks");
 		hns3_interrupt_handler(&rte_eth_devices[hw->data->port_id]);
+		reset_level = hns3_get_reset_level(hns, &hw->reset.pending);
+		if (reset_level == HNS3_NONE_RESET) {
+			hns3_err(hw, "No reset level is set, try IMP reset");
+			hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending);
+		}
 	}
 	rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_NONE);
 
diff --git a/drivers/net/hns3/hns3_ethdev_vf.c b/drivers/net/hns3/hns3_ethdev_vf.c
index 121beb58d..1e9acd4f9 100644
--- a/drivers/net/hns3/hns3_ethdev_vf.c
+++ b/drivers/net/hns3/hns3_ethdev_vf.c
@@ -9,6 +9,8 @@
 #include <inttypes.h>
 #include <unistd.h>
 #include <arpa/inet.h>
+#include <linux/pci_regs.h>
+
 #include <rte_alarm.h>
 #include <rte_atomic.h>
 #include <rte_bus_pci.h>
@@ -24,6 +26,7 @@
 #include <rte_io.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_vfio.h>
 
 #include "hns3_ethdev.h"
 #include "hns3_logs.h"
@@ -56,6 +59,81 @@ static enum hns3_reset_level hns3vf_get_reset_level(struct hns3_hw *hw,
 static int hns3vf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 static int hns3vf_dev_configure_vlan(struct rte_eth_dev *dev);
 
+/* set PCI bus mastering */
+static void
+hns3vf_set_bus_master(const struct rte_pci_device *device, bool op)
+{
+	uint16_t reg;
+
+	rte_pci_read_config(device, &reg, sizeof(reg), PCI_COMMAND);
+
+	if (op)
+		/* set the master bit */
+		reg |= PCI_COMMAND_MASTER;
+	else
+		reg &= ~(PCI_COMMAND_MASTER);
+
+	rte_pci_write_config(device, &reg, sizeof(reg), PCI_COMMAND);
+}
+
+/**
+ * hns3vf_find_pci_capability - lookup a capability in the PCI capability list
+ * @cap: the capability
+ *
+ * Return the address of the given capability within the PCI capability list.
+ */
+static inline int
+hns3vf_find_pci_capability(const struct rte_pci_device *device, int cap)
+{
+#define MAX_PCIE_CAPABILITY 48
+	uint16_t status;
+	uint8_t pos;
+	uint8_t id;
+	int ttl;
+
+	rte_pci_read_config(device, &status, sizeof(status), PCI_STATUS);
+	if (!(status & PCI_STATUS_CAP_LIST))
+		return 0;
+
+	ttl = MAX_PCIE_CAPABILITY;
+	rte_pci_read_config(device, &pos, sizeof(pos), PCI_CAPABILITY_LIST);
+	while (ttl-- && pos >= PCI_STD_HEADER_SIZEOF) {
+		rte_pci_read_config(device, &id, sizeof(id),
+				    (pos + PCI_CAP_LIST_ID));
+
+		if (id == 0xFF)
+			break;
+
+		if (id == cap)
+			return (int)pos;
+
+		rte_pci_read_config(device, &pos, sizeof(pos),
+				    (pos + PCI_CAP_LIST_NEXT));
+	}
+	return 0;
+}
+
+static int
+hns3vf_enable_msix(const struct rte_pci_device *device, bool op)
+{
+	uint16_t control;
+	int pos;
+
+	pos = hns3vf_find_pci_capability(device, PCI_CAP_ID_MSIX);
+	if (pos) {
+		rte_pci_read_config(device, &control, sizeof(control),
+				    (pos + PCI_MSIX_FLAGS));
+		if (op)
+			control |= PCI_MSIX_FLAGS_ENABLE;
+		else
+			control &= ~PCI_MSIX_FLAGS_ENABLE;
+		rte_pci_write_config(device, &control, sizeof(control),
+				     (pos + PCI_MSIX_FLAGS));
+		return 0;
+	}
+	return -1;
+}
+
 static int
 hns3vf_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
 		    __attribute__ ((unused)) uint32_t idx,
@@ -1308,9 +1386,30 @@ hns3vf_wait_hardware_ready(struct hns3_adapter *hns)
 	struct hns3_wait_data *wait_data = hw->reset.wait_data;
 	struct timeval tv;
 
-	if (wait_data->result == HNS3_WAIT_SUCCESS)
-		return 0;
-	else if (wait_data->result == HNS3_WAIT_TIMEOUT) {
+	if (wait_data->result == HNS3_WAIT_SUCCESS) {
+		/*
+		 * After vf reset is ready, the PF may not have completed
+		 * the reset processing. The vf sending mbox to PF may fail
+		 * during the pf reset, so it is better to add extra delay.
+		 */
+		if (hw->reset.level == HNS3_VF_FUNC_RESET ||
+		    hw->reset.level == HNS3_FLR_RESET)
+			return 0;
+		/* Reset retry process, no need to add extra delay. */
+		if (hw->reset.attempts)
+			return 0;
+		if (wait_data->check_completion == NULL)
+			return 0;
+
+		wait_data->check_completion = NULL;
+		wait_data->interval = 1 * MSEC_PER_SEC * USEC_PER_MSEC;
+		wait_data->count = 1;
+		wait_data->result = HNS3_WAIT_REQUEST;
+		rte_eal_alarm_set(wait_data->interval, hns3_wait_callback,
+				  wait_data);
+		hns3_warn(hw, "hardware is ready, delay 1 sec for PF reset complete");
+		return -EAGAIN;
+	} else if (wait_data->result == HNS3_WAIT_TIMEOUT) {
 		gettimeofday(&tv, NULL);
 		hns3_warn(hw, "Reset step4 hardware not ready after reset time=%ld.%.6ld",
 			  tv.tv_sec, tv.tv_usec);
@@ -1473,6 +1572,11 @@ hns3vf_reset_service(void *param)
 		rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_REQUESTED);
 		hns3_err(hw, "Handling interrupts in delayed tasks");
 		hns3vf_interrupt_handler(&rte_eth_devices[hw->data->port_id]);
+		reset_level = hns3vf_get_reset_level(hw, &hw->reset.pending);
+		if (reset_level == HNS3_NONE_RESET) {
+			hns3_err(hw, "No reset level is set, try global reset");
+			hns3_atomic_set_bit(HNS3_VF_RESET, &hw->reset.pending);
+		}
 	}
 	rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_NONE);
 
@@ -1498,14 +1602,35 @@ hns3vf_reset_service(void *param)
 static int
 hns3vf_reinit_dev(struct hns3_adapter *hns)
 {
+	struct rte_eth_dev *eth_dev = &rte_eth_devices[hns->hw.data->port_id];
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 	struct hns3_hw *hw = &hns->hw;
 	int ret;
 
+	if (hw->reset.level == HNS3_VF_FULL_RESET) {
+		rte_intr_disable(&pci_dev->intr_handle);
+		hns3vf_set_bus_master(pci_dev, true);
+	}
+
 	/* Firmware command initialize */
 	ret = hns3_cmd_init(hw);
 	if (ret) {
 		hns3_err(hw, "Failed to init cmd: %d", ret);
-		return ret;
+		goto err_cmd_init;
+	}
+
+	if (hw->reset.level == HNS3_VF_FULL_RESET) {
+		/*
+		 * UIO enables msix by writing the pcie configuration space
+		 * vfio_pci enables msix in rte_intr_enable.
+		 */
+		if (pci_dev->kdrv == RTE_KDRV_IGB_UIO ||
+		    pci_dev->kdrv == RTE_KDRV_UIO_GENERIC) {
+			if (hns3vf_enable_msix(pci_dev, true))
+				hns3_err(hw, "Failed to enable msix");
+		}
+
+		rte_intr_enable(&pci_dev->intr_handle);
 	}
 
 	ret = hns3_reset_all_queues(hns);
@@ -1522,6 +1647,8 @@ hns3vf_reinit_dev(struct hns3_adapter *hns)
 
 	return 0;
 
+err_cmd_init:
+	hns3vf_set_bus_master(pci_dev, false);
 err_init:
 	hns3_cmd_uninit(hw);
 	return ret;
diff --git a/drivers/net/hns3/hns3_intr.c b/drivers/net/hns3/hns3_intr.c
index 9e2d81156..6c3ebd3ee 100644
--- a/drivers/net/hns3/hns3_intr.c
+++ b/drivers/net/hns3/hns3_intr.c
@@ -890,11 +890,12 @@ hns3_reset_err_handle(struct hns3_adapter *hns)
 	hns3_warn(hw, "%s reset fail fail_cnt:%" PRIx64 " success_cnt:%" PRIx64
 		  " global_cnt:%" PRIx64 " imp_cnt:%" PRIx64
 		  " request_cnt:%" PRIx64 " exec_cnt:%" PRIx64
-		  " merge_cnt:%" PRIx64,
+		  " merge_cnt:%" PRIx64 "adapter_state:%d",
 		  reset_string[hw->reset.level], hw->reset.stats.fail_cnt,
 		  hw->reset.stats.success_cnt, hw->reset.stats.global_cnt,
 		  hw->reset.stats.imp_cnt, hw->reset.stats.request_cnt,
-		  hw->reset.stats.exec_cnt, hw->reset.stats.merge_cnt);
+		  hw->reset.stats.exec_cnt, hw->reset.stats.merge_cnt,
+		  hw->adapter_state);
 
 	/* IMP no longer waiting the ready flag */
 	hns3_notify_reset_ready(hw, true);
-- 
2.23.0