[dpdk-dev] [RFC PATCH 09/17] virtio: Fix how states are handled during initialization

[Ouyang Changchun <changchun.ouyang@intel.com>]

Change order of initialiazation to match Linux kernel.
Don't blow away control queue by doing reset when stopped.

Calling dev_stop then dev_start would not work.
Dev_stop was calling virtio reset and that would clear all queues
and clear all feature negotiation.
Resolved by only doing reset on device removal.

Signed-off-by: Changchun Ouyang <changchun.ouyang@intel.com>
Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
---
 lib/librte_pmd_virtio/virtio_ethdev.c | 58 ++++++++++++++++++++---------------
 lib/librte_pmd_virtio/virtio_pci.c    | 10 ++----
 lib/librte_pmd_virtio/virtio_pci.h    |  3 +-
 3 files changed, 37 insertions(+), 34 deletions(-)

diff --git a/lib/librte_pmd_virtio/virtio_ethdev.c b/lib/librte_pmd_virtio/virtio_ethdev.c
index b7f65b9..a07f4ca 100644
--- a/lib/librte_pmd_virtio/virtio_ethdev.c
+++ b/lib/librte_pmd_virtio/virtio_ethdev.c
@@ -398,9 +398,14 @@ virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx,
 static void
 virtio_dev_close(struct rte_eth_dev *dev)
 {
+	struct virtio_hw *hw = dev->data->dev_private;
+
 	PMD_INIT_LOG(DEBUG, "virtio_dev_close");
 
-	virtio_dev_stop(dev);
+	/* reset the NIC */
+	vtpci_irq_config(hw, VIRTIO_MSI_NO_VECTOR);
+	vtpci_reset(hw);
+	virtio_dev_free_mbufs(dev);
 }
 
 static void
@@ -889,6 +894,9 @@ eth_virtio_dev_init(__rte_unused struct eth_driver *eth_drv,
 	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
 		return 0;
 
+	/* Tell the host we've noticed this device. */
+	vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
+
 	pci_dev = eth_dev->pci_dev;
 	if (virtio_resource_init(pci_dev) < 0)
 		return -1;
@@ -899,9 +907,6 @@ eth_virtio_dev_init(__rte_unused struct eth_driver *eth_drv,
 	/* Reset the device although not necessary at startup */
 	vtpci_reset(hw);
 
-	/* Tell the host we've noticed this device. */
-	vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
-
 	/* Tell the host we've known how to drive the device. */
 	vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
 	virtio_negotiate_features(hw);
@@ -990,6 +995,9 @@ eth_virtio_dev_init(__rte_unused struct eth_driver *eth_drv,
 	/* Setup interrupt callback  */
 	rte_intr_callback_register(&pci_dev->intr_handle,
 				   virtio_interrupt_handler, eth_dev);
+
+	virtio_dev_cq_start(eth_dev);
+
 	return 0;
 }
 
@@ -1044,7 +1052,6 @@ virtio_dev_configure(struct rte_eth_dev *dev)
 {
 	const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
 	struct virtio_hw *hw = dev->data->dev_private;
-	int ret;
 
 	PMD_INIT_LOG(DEBUG, "configure");
 
@@ -1055,11 +1062,12 @@ virtio_dev_configure(struct rte_eth_dev *dev)
 
 	hw->vlan_strip = rxmode->hw_vlan_strip;
 
-	ret = vtpci_irq_config(hw, 0);
-	if (ret != 0)
+	if (vtpci_irq_config(hw, 0) == VIRTIO_MSI_NO_VECTOR) {
 		PMD_DRV_LOG(ERR, "failed to set config vector");
+		return -EBUSY;
+	}
 
-	return ret;
+	return 0;
 }
 
 
@@ -1069,17 +1077,6 @@ virtio_dev_start(struct rte_eth_dev *dev)
 	uint16_t nb_queues, i;
 	struct virtio_hw *hw = dev->data->dev_private;
 
-	/* Tell the host we've noticed this device. */
-	vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
-
-	/* Tell the host we've known how to drive the device. */
-	vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
-
-	virtio_dev_cq_start(dev);
-
-	/* Do final configuration before rx/tx engine starts */
-	virtio_dev_rxtx_start(dev);
-
 	/* check if lsc interrupt feature is enabled */
 	if (dev->data->dev_conf.intr_conf.lsc) {
 		if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
@@ -1096,8 +1093,16 @@ virtio_dev_start(struct rte_eth_dev *dev)
 	/* Initialize Link state */
 	virtio_dev_link_update(dev, 0);
 
+	/* On restart after stop do not touch queues */
+	if (hw->started)
+		return 0;
+
 	vtpci_reinit_complete(hw);
 
+	/* Do final configuration before rx/tx engine starts */
+	virtio_dev_rxtx_start(dev);
+	hw->started = 1;
+
 	/*Notify the backend
 	 *Otherwise the tap backend might already stop its queue due to fullness.
 	 *vhost backend will have no chance to be waked up
@@ -1168,17 +1173,20 @@ static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
 }
 
 /*
- * Stop device: disable rx and tx functions to allow for reconfiguring.
+ * Stop device: disable interrupt and mark link down
  */
 static void
 virtio_dev_stop(struct rte_eth_dev *dev)
 {
-	struct virtio_hw *hw = dev->data->dev_private;
+	struct rte_eth_link link;
 
-	/* reset the NIC */
-	vtpci_irq_config(hw, 0);
-	vtpci_reset(hw);
-	virtio_dev_free_mbufs(dev);
+	PMD_INIT_LOG(DEBUG, "stop");
+
+	if (dev->data->dev_conf.intr_conf.lsc)
+		rte_intr_disable(&dev->pci_dev->intr_handle);
+
+	memset(&link, 0, sizeof(link));
+	virtio_dev_atomic_write_link_status(dev, &link);
 }
 
 static int
diff --git a/lib/librte_pmd_virtio/virtio_pci.c b/lib/librte_pmd_virtio/virtio_pci.c
index 6d51032..b099e4f 100644
--- a/lib/librte_pmd_virtio/virtio_pci.c
+++ b/lib/librte_pmd_virtio/virtio_pci.c
@@ -137,15 +137,9 @@ vtpci_isr(struct virtio_hw *hw)
 
 
 /* Enable one vector (0) for Link State Intrerrupt */
-int
+uint16_t
 vtpci_irq_config(struct virtio_hw *hw, uint16_t vec)
 {
 	VIRTIO_WRITE_REG_2(hw, VIRTIO_MSI_CONFIG_VECTOR, vec);
-	vec = VIRTIO_READ_REG_2(hw, VIRTIO_MSI_CONFIG_VECTOR);
-	if (vec == VIRTIO_MSI_NO_VECTOR) {
-		PMD_DRV_LOG(ERR, "failed to set config vector");
-		return -EBUSY;
-	}
-
-	return 0;
+	return VIRTIO_READ_REG_2(hw, VIRTIO_MSI_CONFIG_VECTOR);
 }
diff --git a/lib/librte_pmd_virtio/virtio_pci.h b/lib/librte_pmd_virtio/virtio_pci.h
index 6d93fac..0a4b578 100644
--- a/lib/librte_pmd_virtio/virtio_pci.h
+++ b/lib/librte_pmd_virtio/virtio_pci.h
@@ -170,6 +170,7 @@ struct virtio_hw {
 	uint16_t    vtnet_hdr_size;
 	uint8_t	    vlan_strip;
 	uint8_t	    use_msix;
+	uint8_t     started;
 	uint8_t     mac_addr[ETHER_ADDR_LEN];
 };
 
@@ -266,6 +267,6 @@ void vtpci_read_dev_config(struct virtio_hw *, uint64_t, void *, int);
 
 uint8_t vtpci_isr(struct virtio_hw *);
 
-int vtpci_irq_config(struct virtio_hw *, uint16_t);
+uint16_t vtpci_irq_config(struct virtio_hw *, uint16_t);
 
 #endif /* _VIRTIO_PCI_H_ */
-- 
1.8.4.2