[PATCH 28/35] net/ionic: add Q-in-CMB option controlled by devarg

[Andrew Boyer <andrew.boyer@amd.com>]

When 'ionic_cmb' is set to '1', queue memory will be allocated from
the device's onboard memory (Controller Memory Buffer). In some
configurations, this will dramatically reduce packet latency and
increase PPS.

Add the WC_ACTIVATE flag to the PCI driver flags.
Write combining must be enabled to achieve the maximum PPS.

When the queue is in the CMB, descriptors cannot be prefetched.

Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
Signed-off-by: Neel Patel <neel.patel@amd.com>
---
 doc/guides/nics/ionic.rst              | 12 ++++++
 doc/guides/rel_notes/release_22_11.rst |  1 +
 drivers/net/ionic/ionic.h              |  5 +++
 drivers/net/ionic/ionic_dev.h          |  3 ++
 drivers/net/ionic/ionic_dev_pci.c      | 60 +++++++++++++++++++++++++-
 drivers/net/ionic/ionic_ethdev.c       |  9 ++++
 drivers/net/ionic/ionic_lif.c          | 38 ++++++++++++++++
 drivers/net/ionic/ionic_lif.h          |  2 +
 drivers/net/ionic/ionic_rxtx.c         |  9 ++--
 9 files changed, 135 insertions(+), 4 deletions(-)

diff --git a/doc/guides/nics/ionic.rst b/doc/guides/nics/ionic.rst
index 24b57fc0f5..2713771e4f 100644
--- a/doc/guides/nics/ionic.rst
+++ b/doc/guides/nics/ionic.rst
@@ -32,6 +32,18 @@ The ionic PMD requires firmware which supports 16 segment transmit SGLs.
 This support was added prior to version 1.0. For help upgrading older versions,
 please contact AMD Pensando support.
 
+Runtime Configuration
+---------------------
+
+- ``Queue in CMB support`` (default ``0``)
+
+  Queue memory can be allocated from the Controller Memory Buffer (CMB) using
+  the ``ionic_cmb`` ``devargs`` parameter.
+
+  For example::
+
+    -a 0000:b5:00.0,ionic_cmb=1
+
 Building DPDK
 -------------
 
diff --git a/doc/guides/rel_notes/release_22_11.rst b/doc/guides/rel_notes/release_22_11.rst
index d2e82979e6..974400d0a6 100644
--- a/doc/guides/rel_notes/release_22_11.rst
+++ b/doc/guides/rel_notes/release_22_11.rst
@@ -87,6 +87,7 @@ New Features
   * Added support for mbuf fast free.
   * Added support for advertising packet types.
   * Added support for descriptor status functions.
+  * Added Q-in-CMB feature controlled by devarg ionic_cmb.
 
 Removed Items
 -------------
diff --git a/drivers/net/ionic/ionic.h b/drivers/net/ionic/ionic.h
index 6bfab623f7..1a38e01bc9 100644
--- a/drivers/net/ionic/ionic.h
+++ b/drivers/net/ionic/ionic.h
@@ -24,6 +24,9 @@
 #define IONIC_DEV_ID_ETH_VF		0x1003
 #define IONIC_DEV_ID_ETH_MGMT		0x1004
 
+/* Devargs */
+#define PMD_IONIC_CMB_KVARG		"ionic_cmb"
+
 enum ionic_mac_type {
 	IONIC_MAC_UNKNOWN = 0,
 	IONIC_MAC_CAPRI,
@@ -62,9 +65,11 @@ struct ionic_adapter {
 	uint32_t link_speed;
 	uint32_t nintrs;
 	bool intrs[IONIC_INTR_CTRL_REGS_MAX];
+	bool q_in_cmb;
 	bool link_up;
 	char fw_version[IONIC_DEVINFO_FWVERS_BUFLEN];
 	void *bus_dev;
+	uint64_t cmb_offset;
 };
 
 /** ionic_admin_ctx - Admin command context.
diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h
index 6a80ebc71b..6b0540f615 100644
--- a/drivers/net/ionic/ionic_dev.h
+++ b/drivers/net/ionic/ionic_dev.h
@@ -176,9 +176,12 @@ struct ionic_adapter;
 struct ionic_qcq;
 struct rte_mempool;
 struct rte_eth_dev;
+struct rte_devargs;
 
 struct ionic_dev_intf {
 	int  (*setup)(struct ionic_adapter *adapter);
+	int  (*devargs)(struct ionic_adapter *adapter,
+			struct rte_devargs *devargs);
 	void (*copy_bus_info)(struct ionic_adapter *adapter,
 			struct rte_eth_dev *eth_dev);
 	int  (*configure_intr)(struct ionic_adapter *adapter);
diff --git a/drivers/net/ionic/ionic_dev_pci.c b/drivers/net/ionic/ionic_dev_pci.c
index 1735fa9b17..6c0cf18feb 100644
--- a/drivers/net/ionic/ionic_dev_pci.c
+++ b/drivers/net/ionic/ionic_dev_pci.c
@@ -15,6 +15,7 @@
 #include <rte_eal.h>
 #include <ethdev_pci.h>
 #include <rte_dev.h>
+#include <rte_kvargs.h>
 
 #include "ionic.h"
 #include "ionic_if.h"
@@ -92,6 +93,58 @@ ionic_pci_setup(struct ionic_adapter *adapter)
 	return 0;
 }
 
+const char *ionic_pci_devargs_arr[] = {
+	PMD_IONIC_CMB_KVARG,
+	NULL,
+};
+
+static int
+ionic_pci_devarg_cmb(const char *key __rte_unused, const char *val, void *arg)
+{
+	struct ionic_adapter *adapter = arg;
+
+	if (!strcmp(val, "1")) {
+		IONIC_PRINT(NOTICE, "%s enabled", PMD_IONIC_CMB_KVARG);
+		adapter->q_in_cmb = true;
+	} else if (!strcmp(val, "0")) {
+		IONIC_PRINT(DEBUG, "%s disabled (default)",
+			PMD_IONIC_CMB_KVARG);
+	} else {
+		IONIC_PRINT(ERR, "%s=%s invalid, use 1 or 0",
+			PMD_IONIC_CMB_KVARG, val);
+		return -ERANGE;
+	}
+
+	return 0;
+}
+
+static int
+ionic_pci_devargs(struct ionic_adapter *adapter, struct rte_devargs *devargs)
+{
+	struct rte_kvargs *kvlist;
+	int err = 0;
+
+	if (!devargs)
+		return 0;
+
+	kvlist = rte_kvargs_parse(devargs->args, ionic_pci_devargs_arr);
+	if (!kvlist) {
+		IONIC_PRINT(ERR, "Couldn't parse args '%s'", devargs->args);
+		return -EINVAL;
+	}
+
+	if (rte_kvargs_count(kvlist, PMD_IONIC_CMB_KVARG) == 1) {
+		err = rte_kvargs_process(kvlist, PMD_IONIC_CMB_KVARG,
+				ionic_pci_devarg_cmb, adapter);
+		if (err < 0)
+			goto free_kvlist;
+	}
+
+free_kvlist:
+	rte_kvargs_free(kvlist);
+	return err;
+}
+
 static void
 ionic_pci_copy_bus_info(struct ionic_adapter *adapter,
 	struct rte_eth_dev *eth_dev)
@@ -160,6 +213,7 @@ ionic_pci_unconfigure_intr(struct ionic_adapter *adapter)
 
 static const struct ionic_dev_intf ionic_pci_intf = {
 	.setup = ionic_pci_setup,
+	.devargs = ionic_pci_devargs,
 	.copy_bus_info = ionic_pci_copy_bus_info,
 	.configure_intr = ionic_pci_configure_intr,
 	.unconfigure_intr = ionic_pci_unconfigure_intr,
@@ -206,7 +260,8 @@ eth_ionic_pci_remove(struct rte_pci_device *pci_dev)
 
 static struct rte_pci_driver rte_pci_ionic_pmd = {
 	.id_table = pci_id_ionic_map,
-	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
+	.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
+			RTE_PCI_DRV_WC_ACTIVATE,
 	.probe = eth_ionic_pci_probe,
 	.remove = eth_ionic_pci_remove,
 };
@@ -214,3 +269,6 @@ static struct rte_pci_driver rte_pci_ionic_pmd = {
 RTE_PMD_REGISTER_PCI(net_ionic_pci, rte_pci_ionic_pmd);
 RTE_PMD_REGISTER_PCI_TABLE(net_ionic_pci, pci_id_ionic_map);
 RTE_PMD_REGISTER_KMOD_DEP(net_ionic_pci, "* igb_uio | uio_pci_generic | vfio-pci");
+RTE_PMD_REGISTER_PARAM_STRING(net_ionic_pci,
+	PMD_IONIC_CMB_KVARG "=<0|1>"
+);
diff --git a/drivers/net/ionic/ionic_ethdev.c b/drivers/net/ionic/ionic_ethdev.c
index cf74600f22..a6e7c7fa9f 100644
--- a/drivers/net/ionic/ionic_ethdev.c
+++ b/drivers/net/ionic/ionic_ethdev.c
@@ -1118,6 +1118,15 @@ eth_ionic_dev_probe(void *bus_dev, struct rte_device *rte_dev,
 
 	adapter->intf = intf;
 
+	/* Parse device arguments */
+	if (adapter->intf->devargs) {
+		err = (*adapter->intf->devargs)(adapter, rte_dev->devargs);
+		if (err) {
+			IONIC_PRINT(ERR, "Cannot parse device arguments");
+			goto err_free_adapter;
+		}
+	}
+
 	/* Discover ionic dev resources */
 	err = ionic_setup(adapter);
 	if (err) {
diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c
index ac9b69fc70..bb107b30e9 100644
--- a/drivers/net/ionic/ionic_lif.c
+++ b/drivers/net/ionic/ionic_lif.c
@@ -660,6 +660,21 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 		ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 	}
 
+	if (flags & IONIC_QCQ_F_CMB) {
+		/* alloc descriptor ring from nic memory */
+		if (lif->adapter->cmb_offset + q_size >
+				lif->adapter->bars.bar[2].len) {
+			IONIC_PRINT(ERR, "Cannot reserve queue from NIC mem");
+			return -ENOMEM;
+		}
+		q_base = (void *)
+			((uintptr_t)lif->adapter->bars.bar[2].vaddr +
+			 (uintptr_t)lif->adapter->cmb_offset);
+		/* CMB PA is a relative address */
+		q_base_pa = lif->adapter->cmb_offset;
+		lif->adapter->cmb_offset += q_size;
+	}
+
 	IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 		"SG-base-PA = %#jx",
 		q_base_pa, cq_base_pa, sg_base_pa);
@@ -744,6 +759,8 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index,
 	int err;
 
 	flags = IONIC_QCQ_F_SG;
+	if (lif->state & IONIC_LIF_F_Q_IN_CMB)
+		flags |= IONIC_QCQ_F_CMB;
 
 	seg_size = rte_pktmbuf_data_room_size(mb_pool);
 
@@ -806,6 +823,8 @@ ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index,
 	int err;
 
 	flags = IONIC_QCQ_F_SG;
+	if (lif->state & IONIC_LIF_F_Q_IN_CMB)
+		flags |= IONIC_QCQ_F_CMB;
 
 	num_segs_fw = IONIC_TX_MAX_SG_ELEMS_V1 + 1;
 
@@ -994,6 +1013,19 @@ ionic_lif_alloc(struct ionic_lif *lif)
 		return -ENXIO;
 	}
 
+	if (adapter->q_in_cmb) {
+		if (adapter->bars.num_bars >= 3 &&
+		    lif->qtype_info[IONIC_QTYPE_RXQ].version >= 2 &&
+		    lif->qtype_info[IONIC_QTYPE_TXQ].version >= 3) {
+			IONIC_PRINT(INFO, "%s enabled on %s",
+				PMD_IONIC_CMB_KVARG, lif->name);
+			lif->state |= IONIC_LIF_F_Q_IN_CMB;
+		} else {
+			IONIC_PRINT(ERR, "%s not supported on %s, disabled",
+				PMD_IONIC_CMB_KVARG, lif->name);
+		}
+	}
+
 	IONIC_PRINT(DEBUG, "Allocating Lif Info");
 
 	rte_spinlock_init(&lif->adminq_lock);
@@ -1537,6 +1569,9 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
 	};
 	int err;
 
+	if (txq->flags & IONIC_QCQ_F_CMB)
+		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+
 	IONIC_PRINT(DEBUG, "txq_init.index %d", q->index);
 	IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "", q->base_pa);
 	IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
@@ -1588,6 +1623,9 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
 	};
 	int err;
 
+	if (rxq->flags & IONIC_QCQ_F_CMB)
+		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+
 	IONIC_PRINT(DEBUG, "rxq_init.index %d", q->index);
 	IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "", q->base_pa);
 	IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h
index 5231909213..ec9cb24a61 100644
--- a/drivers/net/ionic/ionic_lif.h
+++ b/drivers/net/ionic/ionic_lif.h
@@ -49,6 +49,7 @@ struct ionic_rx_stats {
 #define IONIC_QCQ_F_INITED	BIT(0)
 #define IONIC_QCQ_F_SG		BIT(1)
 #define IONIC_QCQ_F_DEFERRED	BIT(4)
+#define IONIC_QCQ_F_CMB		BIT(5)
 #define IONIC_QCQ_F_CSUM_L3	BIT(7)
 #define IONIC_QCQ_F_CSUM_UDP	BIT(8)
 #define IONIC_QCQ_F_CSUM_TCP	BIT(9)
@@ -126,6 +127,7 @@ struct ionic_qtype_info {
 #define IONIC_LIF_F_LINK_CHECK_NEEDED	BIT(1)
 #define IONIC_LIF_F_UP			BIT(2)
 #define IONIC_LIF_F_FW_RESET		BIT(3)
+#define IONIC_LIF_F_Q_IN_CMB		BIT(4)
 
 #define IONIC_LIF_NAME_MAX_SZ		(32)
 
diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c
index 4f84fa7df1..fceb8333f6 100644
--- a/drivers/net/ionic/ionic_rxtx.c
+++ b/drivers/net/ionic/ionic_rxtx.c
@@ -600,7 +600,8 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	int err;
 
 	struct ionic_txq_desc *desc_base = q->base;
-	rte_prefetch0(&desc_base[q->head_idx]);
+	if (!(txq->flags & IONIC_QCQ_F_CMB))
+		rte_prefetch0(&desc_base[q->head_idx]);
 	rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx));
 
 	if (tx_pkts) {
@@ -619,7 +620,8 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 	while (nb_tx < nb_pkts) {
 		uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
-		rte_prefetch0(&desc_base[next_idx]);
+		if (!(txq->flags & IONIC_QCQ_F_CMB))
+			rte_prefetch0(&desc_base[next_idx]);
 		rte_prefetch0(IONIC_INFO_PTR(q, next_idx));
 
 		if (nb_tx + 1 < nb_pkts) {
@@ -1172,7 +1174,8 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 		/* Prefetch 4 x 16B comp */
 		rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
 		/* Prefetch 4 x 16B descriptors */
-		rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
+		if (!(rxq->flags & IONIC_QCQ_F_CMB))
+			rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
 
 		ionic_rx_clean_one(rxq, cq_desc, rx_svc);
 
-- 
2.17.1