[PATCH] net/gve: support for gVNIC PCI revision 1 and later

[PATCH] net/gve: support for gVNIC PCI revision 1 and later

[Joshua Washington]

gVNIC PCI revisions greater than 0 have support for setting the admin
queue by its full address instead of using the PFN, which can be
problematic if the system page size is not actually 4K as the device
assumes. Beyond that, a new device/driver reset mechanism is introduced.
Originally, a device reset was triggered by writing 0 to the admin queue
PFN. Newer device reivisons set a status flag, instead.

Signed-off-by: Joshua Washington <joshwash@google.com>
Reviewed-by: Ziwei Xiao <ziweixiao@google.com>
---
 .mailmap                            |  1 +
 drivers/net/gve/base/gve.h          |  4 ++
 drivers/net/gve/base/gve_adminq.c   | 59 +++++++++++++++++++++++------
 drivers/net/gve/base/gve_osdep.h    | 18 +++++++++
 drivers/net/gve/base/gve_register.h | 10 +++++
 5 files changed, 80 insertions(+), 12 deletions(-)

diff --git a/.mailmap b/.mailmap
index d8439b79ce..f18f6eaefb 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1831,6 +1831,7 @@ Zhiyong Yang <zhiyong.yang@intel.com>
 Zhuobin Huang <zobin1999@gmail.com>
 Zi Hu <huzilucky@gmail.com>
 Zijie Pan <zijie.pan@6wind.com>
+Ziwei Xiao <ziweixiao@google.com>
 Ziyang Xuan <xuanziyang2@huawei.com>
 Ziye Yang <ziye.yang@intel.com>
 Zoltan Kiss <zoltan.kiss@schaman.hu> <zoltan.kiss@linaro.org>
diff --git a/drivers/net/gve/base/gve.h b/drivers/net/gve/base/gve.h
index 9c58fc4238..0c5f5e9ce4 100644
--- a/drivers/net/gve/base/gve.h
+++ b/drivers/net/gve/base/gve.h
@@ -14,6 +14,8 @@
 #endif
 
 #define GVE_DEV_ID		0x0042
+#define GVE_PCI_REV_OFFSET	0x8
+#define GVE_PCI_REV_SIZE	1
 
 #define GVE_REG_BAR		0
 #define GVE_DB_BAR		2
@@ -24,6 +26,8 @@
 /* PTYPEs are always 10 bits. */
 #define GVE_NUM_PTYPES		1024
 
+#define GVE_ADMINQ_BUFFER_SIZE	4096
+
 struct gve_irq_db {
 	rte_be32_t id;
 } ____cacheline_aligned;
diff --git a/drivers/net/gve/base/gve_adminq.c b/drivers/net/gve/base/gve_adminq.c
index bcb983e4a0..2c5cfa2aa1 100644
--- a/drivers/net/gve/base/gve_adminq.c
+++ b/drivers/net/gve/base/gve_adminq.c
@@ -196,6 +196,16 @@ gve_process_device_options(struct gve_priv *priv,
 	return 0;
 }
 
+static uint8_t
+gve_get_pci_revision_id(struct gve_priv *priv)
+{
+	uint8_t rev_id;
+
+	rte_pci_read_config(priv->pci_dev, &rev_id, GVE_PCI_REV_SIZE,
+			    GVE_PCI_REV_OFFSET);
+	return rev_id;
+}
+
 int gve_adminq_alloc(struct gve_priv *priv)
 {
 	priv->adminq = gve_alloc_dma_mem(&priv->adminq_dma_mem, PAGE_SIZE);
@@ -221,8 +231,19 @@ int gve_adminq_alloc(struct gve_priv *priv)
 	priv->adminq_get_ptype_map_cnt = 0;
 
 	/* Setup Admin queue with the device */
-	iowrite32be(priv->adminq_dma_mem.pa / PAGE_SIZE,
-		    &priv->reg_bar0->adminq_pfn);
+	if (gve_get_pci_revision_id(priv) < 0x1) { /* Use AQ PFN. */
+		iowrite32be(priv->adminq_dma_mem.pa / PAGE_SIZE,
+			    &priv->reg_bar0->adminq_pfn);
+	} else { /* Use full AQ address. */
+		iowrite16be(GVE_ADMINQ_BUFFER_SIZE,
+			    &priv->reg_bar0->adminq_length);
+		iowrite32be(priv->adminq_dma_mem.pa >> 32,
+			    &priv->reg_bar0->adminq_base_address_hi);
+		iowrite32be(priv->adminq_dma_mem.pa,
+			    &priv->reg_bar0->adminq_base_address_lo);
+		iowrite32be(GVE_DRIVER_STATUS_RUN_MASK,
+			    &priv->reg_bar0->driver_status);
+	}
 
 	gve_set_admin_queue_ok(priv);
 	return 0;
@@ -233,16 +254,30 @@ void gve_adminq_release(struct gve_priv *priv)
 	int i = 0;
 
 	/* Tell the device the adminq is leaving */
-	iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
-	while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
-		/* If this is reached the device is unrecoverable and still
-		 * holding memory. Continue looping to avoid memory corruption,
-		 * but WARN so it is visible what is going on.
-		 */
-		if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
-			PMD_DRV_LOG(WARNING, "Unrecoverable platform error!");
-		i++;
-		msleep(GVE_ADMINQ_SLEEP_LEN);
+	if (gve_get_pci_revision_id(priv) < 0x1) {
+		iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
+		while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
+			/* If this is reached the device is unrecoverable and still
+			 * holding memory. Continue looping to avoid memory corruption,
+			 * but WARN so it is visible what is going on.
+			 */
+			if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
+				PMD_DRV_LOG(WARNING,
+					    "Unrecoverable platform error!");
+			i++;
+			msleep(GVE_ADMINQ_SLEEP_LEN);
+		}
+	} else {
+		iowrite32be(GVE_DRIVER_STATUS_RESET_MASK,
+			    &priv->reg_bar0->driver_status);
+		while (!(ioread32be(&priv->reg_bar0->device_status)
+			 & GVE_DEVICE_STATUS_DEVICE_IS_RESET)) {
+			if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
+				PMD_DRV_LOG(WARNING,
+					    "Unrecoverable platform error!");
+			i++;
+			msleep(GVE_ADMINQ_SLEEP_LEN);
+		}
 	}
 	gve_clear_device_rings_ok(priv);
 	gve_clear_device_resources_ok(priv);
diff --git a/drivers/net/gve/base/gve_osdep.h b/drivers/net/gve/base/gve_osdep.h
index d9c82c43e3..5eb33930cd 100644
--- a/drivers/net/gve/base/gve_osdep.h
+++ b/drivers/net/gve/base/gve_osdep.h
@@ -130,18 +130,36 @@ writeb(u8 value, volatile void *addr)
 	rte_write8(value, addr);
 }
 
+static __rte_always_inline void
+writew(u16 value, volatile void *addr)
+{
+	rte_write16(value, addr);
+}
+
 static __rte_always_inline void
 writel(u32 value, volatile void *addr)
 {
 	rte_write32(value, addr);
 }
 
+static __rte_always_inline u16
+ioread16be(const volatile void *addr)
+{
+	return rte_be_to_cpu_16(rte_read16(addr));
+}
+
 static __rte_always_inline u32
 ioread32be(const volatile void *addr)
 {
 	return rte_be_to_cpu_32(rte_read32(addr));
 }
 
+static __rte_always_inline void
+iowrite16be(u16 value, volatile void *addr)
+{
+	writew(rte_cpu_to_be_16(value), addr);
+}
+
 static __rte_always_inline void
 iowrite32be(u32 value, volatile void *addr)
 {
diff --git a/drivers/net/gve/base/gve_register.h b/drivers/net/gve/base/gve_register.h
index c674167f31..d3be475744 100644
--- a/drivers/net/gve/base/gve_register.h
+++ b/drivers/net/gve/base/gve_register.h
@@ -19,11 +19,21 @@ struct gve_registers {
 	__be32	adminq_event_counter;
 	u8	reserved[3];
 	u8	driver_version;
+	__be32	adminq_base_address_hi;
+	__be32	adminq_base_address_lo;
+	__be16	adminq_length;
 };
 
 enum gve_device_status_flags {
 	GVE_DEVICE_STATUS_RESET_MASK		= BIT(1),
 	GVE_DEVICE_STATUS_LINK_STATUS_MASK	= BIT(2),
 	GVE_DEVICE_STATUS_REPORT_STATS_MASK	= BIT(3),
+	GVE_DEVICE_STATUS_DEVICE_IS_RESET	= BIT(4),
 };
+
+enum gve_driver_status_flags {
+	GVE_DRIVER_STATUS_RUN_MASK		= BIT(0),
+	GVE_DRIVER_STATUS_RESET_MASK		= BIT(1),
+};
+
 #endif /* _GVE_REGISTER_H_ */
-- 
2.49.0.604.gff1f9ca942-goog