[PATCH 1/3] net/ark: support for single function with multiple port

[PATCH 1/3] net/ark: support for single function with multiple port

[Ed Czeck]

Support the creation of multiple ports from one ark device via
the use of ark pmd extension.  I.e., one device with q queue can
seen a p ports each with q/p queues.

Add unique dev_private data for each port to manage queue assignment.

This patch repairs a latent issue uncovered during testing.
Fixes: 6799275eeea6 ("net/ark: support virtual functions")
Cc: stable@dpdk.org
Backporting is not requested.

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
 drivers/net/ark/ark_ethdev.c    | 15 ++++++++++++++-
 drivers/net/ark/ark_ethdev_rx.c |  6 +++---
 drivers/net/ark/ark_ethdev_tx.c |  2 +-
 drivers/net/ark/ark_global.h    |  3 +++
 4 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 3ddcc9b461..90d3c8abe6 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -300,6 +300,7 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	int ret;
 	int port_count = 1;
 	int p;
+	uint16_t num_queues;
 	bool rqpacing = false;
 
 	ark->eth_dev = dev;
@@ -427,6 +428,7 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 			ark->user_ext.dev_get_port_count(dev,
 				 ark->user_data[dev->data->port_id]);
 	ark->num_ports = port_count;
+	num_queues = ark_api_num_queues_per_port(ark->mpurx.v, port_count);
 
 	for (p = 0; p < port_count; p++) {
 		struct rte_eth_dev *eth_dev;
@@ -452,7 +454,18 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 		}
 
 		eth_dev->device = &pci_dev->device;
-		eth_dev->data->dev_private = ark;
+		/* Device requires new dev_private data */
+		eth_dev->data->dev_private =
+			rte_zmalloc_socket(name,
+					   sizeof(struct ark_adapter),
+					   RTE_CACHE_LINE_SIZE,
+					   rte_socket_id());
+
+		memcpy(eth_dev->data->dev_private, ark,
+		       sizeof(struct ark_adapter));
+		ark = eth_dev->data->dev_private;
+		ark->qbase = p * num_queues;
+
 		eth_dev->dev_ops = ark->eth_dev->dev_ops;
 		eth_dev->tx_pkt_burst = ark->eth_dev->tx_pkt_burst;
 		eth_dev->rx_pkt_burst = ark->eth_dev->rx_pkt_burst;
diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c
index cbc0416bc2..38bc69dff4 100644
--- a/drivers/net/ark/ark_ethdev_rx.c
+++ b/drivers/net/ark/ark_ethdev_rx.c
@@ -68,7 +68,7 @@ struct ark_rx_queue {
 static int
 eth_ark_rx_hw_setup(struct rte_eth_dev *dev,
 		    struct ark_rx_queue *queue,
-		    uint16_t rx_queue_id __rte_unused, uint16_t rx_queue_idx)
+		    uint16_t rx_queue_idx)
 {
 	rte_iova_t queue_base;
 	rte_iova_t phys_addr_q_base;
@@ -124,7 +124,7 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	uint32_t i;
 	int status;
 
-	int qidx = queue_idx;
+	int qidx = ark->qbase + queue_idx;
 
 	/* We may already be setup, free memory prior to re-allocation */
 	if (dev->data->rx_queues[queue_idx] != NULL) {
@@ -215,7 +215,7 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 	/* MPU Setup */
 	if (status == 0)
-		status = eth_ark_rx_hw_setup(dev, queue, qidx, queue_idx);
+		status = eth_ark_rx_hw_setup(dev, queue, queue_idx);
 
 	if (unlikely(status != 0)) {
 		struct rte_mbuf **mbuf;
diff --git a/drivers/net/ark/ark_ethdev_tx.c b/drivers/net/ark/ark_ethdev_tx.c
index 5940a592a2..4792754f19 100644
--- a/drivers/net/ark/ark_ethdev_tx.c
+++ b/drivers/net/ark/ark_ethdev_tx.c
@@ -229,7 +229,7 @@ eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
 	struct ark_tx_queue *queue;
 	int status;
 
-	int qidx = queue_idx;
+	int qidx = ark->qbase + queue_idx;
 
 	if (!rte_is_power_of_2(nb_desc)) {
 		ARK_PMD_LOG(ERR,
diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h
index 71d0b53e03..2f198edfe4 100644
--- a/drivers/net/ark/ark_global.h
+++ b/drivers/net/ark/ark_global.h
@@ -112,7 +112,10 @@ struct ark_adapter {
 	ark_pkt_chkr_t pc;
 	ark_pkt_dir_t pd;
 
+	/* For single function, multiple ports */
 	int num_ports;
+	uint16_t qbase;
+
 	bool isvf;
 
 	/* Packet generator/checker args */
-- 
2.34.1

[PATCH 2/3] net/ark: remove RQ pacing firmware from PMD

[Ed Czeck]

features and function have been removed from FPGA firmware

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
 drivers/net/ark/ark_ethdev.c | 62 ++++++++------------------------
 drivers/net/ark/ark_global.h |  3 --
 drivers/net/ark/ark_rqp.c    | 70 ------------------------------------
 drivers/net/ark/ark_rqp.h    | 58 ------------------------------
 drivers/net/ark/meson.build  |  1 -
 5 files changed, 15 insertions(+), 179 deletions(-)
 delete mode 100644 drivers/net/ark/ark_rqp.c
 delete mode 100644 drivers/net/ark/ark_rqp.h

diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 90d3c8abe6..306121ba31 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -17,7 +17,6 @@
 #include "ark_mpu.h"
 #include "ark_ddm.h"
 #include "ark_udm.h"
-#include "ark_rqp.h"
 #include "ark_pktdir.h"
 #include "ark_pktgen.h"
 #include "ark_pktchkr.h"
@@ -107,36 +106,32 @@ static const struct rte_pci_id pci_id_ark_map[] = {
  * This structure is used to statically define the capabilities
  * of supported devices.
  * Capabilities:
- *  rqpacing -
- * Some HW variants require that PCIe read-requests be correctly throttled.
- * This is called "rqpacing" and has to do with credit and flow control
- * on certain Arkville implementations.
+ *    isvf -- defined for function id that are virtual
  */
 struct ark_caps {
-	bool rqpacing;
 	bool isvf;
 };
 struct ark_dev_caps {
 	uint32_t  device_id;
 	struct ark_caps  caps;
 };
-#define SET_DEV_CAPS(id, rqp, vf)			\
-	{id, {.rqpacing = rqp, .isvf = vf} }
+#define SET_DEV_CAPS(id, vf)			\
+	{id, {.isvf = vf} }
 
 static const struct ark_dev_caps
 ark_device_caps[] = {
-		     SET_DEV_CAPS(0x100d, true, false),
-		     SET_DEV_CAPS(0x100e, true, false),
-		     SET_DEV_CAPS(0x100f, true, false),
-		     SET_DEV_CAPS(0x1010, false, false),
-		     SET_DEV_CAPS(0x1017, true, false),
-		     SET_DEV_CAPS(0x1018, true, false),
-		     SET_DEV_CAPS(0x1019, true, false),
-		     SET_DEV_CAPS(0x101a, true, false),
-		     SET_DEV_CAPS(0x101b, true, false),
-		     SET_DEV_CAPS(0x101c, true, true),
-		     SET_DEV_CAPS(0x101e, false, false),
-		     SET_DEV_CAPS(0x101f, false, false),
+		     SET_DEV_CAPS(0x100d, false),
+		     SET_DEV_CAPS(0x100e, false),
+		     SET_DEV_CAPS(0x100f, false),
+		     SET_DEV_CAPS(0x1010, false),
+		     SET_DEV_CAPS(0x1017, false),
+		     SET_DEV_CAPS(0x1018, false),
+		     SET_DEV_CAPS(0x1019, false),
+		     SET_DEV_CAPS(0x101a, false),
+		     SET_DEV_CAPS(0x101b, false),
+		     SET_DEV_CAPS(0x101c, true),
+		     SET_DEV_CAPS(0x101e, false),
+		     SET_DEV_CAPS(0x101f, false),
 		     {.device_id = 0,}
 };
 
@@ -301,9 +296,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	int port_count = 1;
 	int p;
 	uint16_t num_queues;
-	bool rqpacing = false;
-
-	ark->eth_dev = dev;
 
 	ARK_PMD_LOG(DEBUG, "\n");
 
@@ -319,7 +311,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	p = 0;
 	while (ark_device_caps[p].device_id != 0) {
 		if (pci_dev->id.device_id == ark_device_caps[p].device_id) {
-			rqpacing = ark_device_caps[p].caps.rqpacing;
 			ark->isvf = ark_device_caps[p].caps.isvf;
 			break;
 		}
@@ -344,12 +335,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	ark->pktgen.v  = (void *)&ark->bar0[ARK_PKTGEN_BASE];
 	ark->pktchkr.v  = (void *)&ark->bar0[ARK_PKTCHKR_BASE];
 
-	if (rqpacing) {
-		ark->rqpacing =
-			(struct ark_rqpace_t *)(ark->bar0 + ARK_RCPACING_BASE);
-	} else {
-		ark->rqpacing = NULL;
-	}
 	ark->started = 0;
 	ark->pkt_dir_v = ARK_PKT_DIR_INIT_VAL;
 
@@ -368,17 +353,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 			    ark->sysctrl.t32[4], __func__);
 		return -1;
 	}
-	if (ark->sysctrl.t32[3] != 0) {
-		if (ark->rqpacing) {
-			if (ark_rqp_lasped(ark->rqpacing)) {
-				ARK_PMD_LOG(ERR, "Arkville Evaluation System - "
-					    "Timer has Expired\n");
-				return -1;
-			}
-			ARK_PMD_LOG(WARNING, "Arkville Evaluation System - "
-				    "Timer is Running\n");
-		}
-	}
 
 	ARK_PMD_LOG(DEBUG,
 		    "HW Sanity test has PASSED, expected constant"
@@ -550,9 +524,6 @@ ark_config_device(struct rte_eth_dev *dev)
 		mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET);
 	}
 
-	if (!ark->isvf && ark->rqpacing)
-		ark_rqp_stats_reset(ark->rqpacing);
-
 	return 0;
 }
 
@@ -709,9 +680,6 @@ eth_ark_dev_close(struct rte_eth_dev *dev)
 	/*
 	 * This should only be called once for the device during shutdown
 	 */
-	if (ark->rqpacing)
-		ark_rqp_dump(ark->rqpacing);
-
 	/* return to power-on state */
 	if (ark->pd)
 		ark_pktdir_setup(ark->pd, ARK_PKT_DIR_INIT_VAL);
diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h
index 2f198edfe4..147b14b6c0 100644
--- a/drivers/net/ark/ark_global.h
+++ b/drivers/net/ark/ark_global.h
@@ -32,7 +32,6 @@
 #define ARK_CMAC_BASE     0x80000
 #define ARK_PKTDIR_BASE   0xa0000
 #define ARK_PKTCHKR_BASE  0x90000
-#define ARK_RCPACING_BASE 0xb0000
 #define ARK_EXTERNAL_BASE 0x100000
 #define ARK_MPU_QOFFSET   0x00100
 #define ARK_MAX_PORTS     RTE_MAX_ETHPORTS
@@ -150,8 +149,6 @@ struct ark_adapter {
 	int started;
 	uint16_t rx_queues;
 	uint16_t tx_queues;
-
-	struct ark_rqpace_t *rqpacing;
 };
 
 typedef uint32_t *ark_t;
diff --git a/drivers/net/ark/ark_rqp.c b/drivers/net/ark/ark_rqp.c
deleted file mode 100644
index efb9730fe6..0000000000
--- a/drivers/net/ark/ark_rqp.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
- */
-
-#include <unistd.h>
-
-#include "ark_rqp.h"
-#include "ark_logs.h"
-
-/* ************************************************************************* */
-void
-ark_rqp_stats_reset(struct ark_rqpace_t *rqp)
-{
-	rqp->stats_clear = 1;
-	/* POR 992 */
-	/* rqp->cpld_max = 992; */
-	/* POR 64 */
-	/* rqp->cplh_max = 64; */
-}
-
-/* ************************************************************************* */
-void
-ark_rqp_dump(struct ark_rqpace_t *rqp)
-{
-	if (rqp->err_count_other || rqp->cmpl_errors)
-		ARK_PMD_LOG(ERR,
-			    "RQP Errors noted: ctrl: %d cplh_hmax %d cpld_max %d"
-			    ARK_SU32
-			    ARK_SU32
-			    ARK_SU32 "\n",
-			    rqp->ctrl, rqp->cplh_max, rqp->cpld_max,
-			    "Error Count", rqp->err_cnt,
-			    "Error General", rqp->err_count_other,
-			    "Cmpl Errors", rqp->cmpl_errors);
-
-	ARK_PMD_LOG(INFO, "RQP Dump: ctrl: %d cplh_hmax %d cpld_max %d"
-		      ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 "\n",
-		      rqp->ctrl, rqp->cplh_max, rqp->cpld_max,
-		      "Error Count", rqp->err_cnt,
-		      "Error General", rqp->err_count_other,
-		      "stall_pS", rqp->stall_ps,
-		      "stall_pS Min", rqp->stall_ps_min,
-		      "stall_pS Max", rqp->stall_ps_max,
-		      "req_pS", rqp->req_ps,
-		      "req_pS Min", rqp->req_ps_min,
-		      "req_pS Max", rqp->req_ps_max,
-		      "req_dWPS", rqp->req_dw_ps,
-		      "req_dWPS Min", rqp->req_dw_ps_min,
-		      "req_dWPS Max", rqp->req_dw_ps_max,
-		      "cpl_pS", rqp->cpl_ps,
-		      "cpl_pS Min", rqp->cpl_ps_min,
-		      "cpl_pS Max", rqp->cpl_ps_max,
-		      "cpl_dWPS", rqp->cpl_dw_ps,
-		      "cpl_dWPS Min", rqp->cpl_dw_ps_min,
-		      "cpl_dWPS Max", rqp->cpl_dw_ps_max,
-		      "cplh pending", rqp->cplh_pending,
-		      "cpld pending", rqp->cpld_pending,
-		      "cplh pending max", rqp->cplh_pending_max,
-		      "cpld pending max", rqp->cpld_pending_max);
-}
-
-int
-ark_rqp_lasped(struct ark_rqpace_t *rqp)
-{
-	return rqp->lasped;
-}
diff --git a/drivers/net/ark/ark_rqp.h b/drivers/net/ark/ark_rqp.h
deleted file mode 100644
index d09f242e1e..0000000000
--- a/drivers/net/ark/ark_rqp.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
- */
-
-#ifndef _ARK_RQP_H_
-#define _ARK_RQP_H_
-
-#include <stdint.h>
-
-#include <rte_memory.h>
-
-/* The RQP or ReQuest Pacer is an internal Arkville hardware module
- * which limits the PCIE data flow to insure correct operation for the
- * particular hardware PCIE endpoint.
- * This module is *not* intended for end-user manipulation, hence
- * there is minimal documentation.
- */
-
-/*
- * RQ Pacing core hardware structure
- * This is an overlay structures to a memory mapped FPGA device.  These
- * structs will never be instantiated in ram memory
- */
-struct ark_rqpace_t {
-	volatile uint32_t ctrl;
-	volatile uint32_t stats_clear;
-	volatile uint32_t cplh_max;
-	volatile uint32_t cpld_max;
-	volatile uint32_t err_cnt;
-	volatile uint32_t stall_ps;
-	volatile uint32_t stall_ps_min;
-	volatile uint32_t stall_ps_max;
-	volatile uint32_t req_ps;
-	volatile uint32_t req_ps_min;
-	volatile uint32_t req_ps_max;
-	volatile uint32_t req_dw_ps;
-	volatile uint32_t req_dw_ps_min;
-	volatile uint32_t req_dw_ps_max;
-	volatile uint32_t cpl_ps;
-	volatile uint32_t cpl_ps_min;
-	volatile uint32_t cpl_ps_max;
-	volatile uint32_t cpl_dw_ps;
-	volatile uint32_t cpl_dw_ps_min;
-	volatile uint32_t cpl_dw_ps_max;
-	volatile uint32_t cplh_pending;
-	volatile uint32_t cpld_pending;
-	volatile uint32_t cplh_pending_max;
-	volatile uint32_t cpld_pending_max;
-	volatile uint32_t err_count_other;
-	char eval[4];
-	volatile int32_t lasped;
-	volatile uint32_t cmpl_errors;
-};
-
-void ark_rqp_dump(struct ark_rqpace_t *rqp);
-void ark_rqp_stats_reset(struct ark_rqpace_t *rqp);
-int ark_rqp_lasped(struct ark_rqpace_t *rqp);
-#endif
diff --git a/drivers/net/ark/meson.build b/drivers/net/ark/meson.build
index 8d87744c22..12b3935b85 100644
--- a/drivers/net/ark/meson.build
+++ b/drivers/net/ark/meson.build
@@ -16,6 +16,5 @@ sources = files(
         'ark_pktchkr.c',
         'ark_pktdir.c',
         'ark_pktgen.c',
-        'ark_rqp.c',
         'ark_udm.c',
 )
-- 
2.34.1

[PATCH 3/3] net/ark: support for large dataroom in FPGA

[Ed Czeck]

Allow for non-standard dataroom sizes for upstream data
motion when using a custom mbuf pool.

- New firmware version for UDM (Upstream Data Mover)
- New layout on ark_rx_meta data

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
 doc/guides/nics/ark.rst         | 1 +
 drivers/net/ark/ark_ethdev_rx.c | 8 ++++++++
 drivers/net/ark/ark_ext.h       | 9 +++++++++
 drivers/net/ark/ark_udm.h       | 5 +++--
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/doc/guides/nics/ark.rst b/doc/guides/nics/ark.rst
index e1683cf782..bcc9f505df 100644
--- a/doc/guides/nics/ark.rst
+++ b/doc/guides/nics/ark.rst
@@ -334,6 +334,7 @@ with Arkville releases 21.05, 21.08 and 21.11. LTS versions of DPDK remain
 compatible with the corresponding Arkville version.  If other combinations
 are required, please contact Atomic Rules support.
 
+* DPDK 23.11 requires Arkville 23.11.
 * DPDK 22.07 requires Arkville 22.07.
 * DPDK 22.03 requires Arkville 22.03.
 * DPDK 21.05 requires Arkville 21.05.
diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c
index 38bc69dff4..24f1c65690 100644
--- a/drivers/net/ark/ark_ethdev_rx.c
+++ b/drivers/net/ark/ark_ethdev_rx.c
@@ -9,6 +9,7 @@
 #include "ark_logs.h"
 #include "ark_mpu.h"
 #include "ark_udm.h"
+#include "ark_ext.h"
 
 #define ARK_RX_META_SIZE 32
 #define ARK_RX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_RX_META_SIZE)
@@ -166,6 +167,13 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	queue->mb_pool = mb_pool;
 	queue->dataroom = rte_pktmbuf_data_room_size(mb_pool) -
 		RTE_PKTMBUF_HEADROOM;
+
+	/* Check pool's private data to confirm pool structure */
+	if (mb_pool->private_data_size != 0) {
+		struct rte_pmd_ark_lmbuf_mempool_priv *pool_priv = rte_mempool_get_priv(mb_pool);
+		if (strncmp(pool_priv->cookie, ARK_MEMPOOL_COOKIE, sizeof(pool_priv->cookie)) == 0)
+			queue->dataroom = pool_priv->dataroom;
+	}
 	queue->headroom = RTE_PKTMBUF_HEADROOM;
 	queue->phys_qid = qidx;
 	queue->queue_index = queue_idx;
diff --git a/drivers/net/ark/ark_ext.h b/drivers/net/ark/ark_ext.h
index d235d0ff85..6d37449195 100644
--- a/drivers/net/ark/ark_ext.h
+++ b/drivers/net/ark/ark_ext.h
@@ -5,6 +5,7 @@
 #ifndef _ARK_EXT_H_
 #define _ARK_EXT_H_
 
+#include <rte_mbuf.h>
 #include <ethdev_driver.h>
 
 /* The following section lists function prototypes for Arkville's
@@ -16,6 +17,14 @@
  * See documentation for compiling and use of extensions.
  */
 
+/* private data optionally attached to mempool for rx */
+struct rte_pmd_ark_lmbuf_mempool_priv {
+	struct rte_pktmbuf_pool_private pool_private;
+	char cookie[4];
+	uint32_t dataroom;
+};
+#define ARK_MEMPOOL_COOKIE "ARK1"
+
 /**
  * Extension prototype, required implementation if extensions are used.
  * Called during device probe to initialize the user structure
diff --git a/drivers/net/ark/ark_udm.h b/drivers/net/ark/ark_udm.h
index f0685c95c7..ec53ec7e79 100644
--- a/drivers/net/ark/ark_udm.h
+++ b/drivers/net/ark/ark_udm.h
@@ -20,7 +20,8 @@
  */
 struct ark_rx_meta {
 	uint32_t user_meta[5];	/* user defined based on fpga code */
-	uint8_t  reserved[10];
+	uint32_t pkt_len32;
+	uint8_t  reserved[6];
 	uint16_t pkt_len;
 } __rte_packed;
 
@@ -33,7 +34,7 @@ struct ark_rx_meta {
 #define ARK_RX_WRITE_TIME_NS 2500
 #define ARK_UDM_SETUP 0
 #define ARK_UDM_MODID 0x4d445500
-#define ARK_UDM_MODVER 0x37313232
+#define ARK_UDM_MODVER 0x37333332
 
 struct ark_udm_setup_t {
 	union {
-- 
2.34.1

Re: [PATCH 2/3] net/ark: remove RQ pacing firmware from PMD

[Ferruh Yigit]

On 10/5/2023 9:52 PM, Ed Czeck wrote:
> features and function have been removed from FPGA firmware
> 

I am always a little confused how you manage the deployment, if a
customer requires RQ pacing, how you manage it, at least should it be
documented in driver documentation that RQ pacing supported before
v23.11, or something like that?
If this doesn't make sense for your deployment model, scratch it, this
is just a reminder if it is useful.


> Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
> ---
>  drivers/net/ark/ark_ethdev.c | 62 ++++++++------------------------
>  drivers/net/ark/ark_global.h |  3 --
>  drivers/net/ark/ark_rqp.c    | 70 ------------------------------------
>  drivers/net/ark/ark_rqp.h    | 58 ------------------------------
>  drivers/net/ark/meson.build  |  1 -
>  5 files changed, 15 insertions(+), 179 deletions(-)
>  delete mode 100644 drivers/net/ark/ark_rqp.c
>  delete mode 100644 drivers/net/ark/ark_rqp.h
> 
> diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
> index 90d3c8abe6..306121ba31 100644
> --- a/drivers/net/ark/ark_ethdev.c
> +++ b/drivers/net/ark/ark_ethdev.c
> @@ -17,7 +17,6 @@
>  #include "ark_mpu.h"
>  #include "ark_ddm.h"
>  #include "ark_udm.h"
> -#include "ark_rqp.h"
>  #include "ark_pktdir.h"
>  #include "ark_pktgen.h"
>  #include "ark_pktchkr.h"
> @@ -107,36 +106,32 @@ static const struct rte_pci_id pci_id_ark_map[] = {
>   * This structure is used to statically define the capabilities
>   * of supported devices.
>   * Capabilities:
> - *  rqpacing -
> - * Some HW variants require that PCIe read-requests be correctly throttled.
> - * This is called "rqpacing" and has to do with credit and flow control
> - * on certain Arkville implementations.
> + *    isvf -- defined for function id that are virtual
>   */
>  struct ark_caps {
> -	bool rqpacing;
>  	bool isvf;
>  };
>  struct ark_dev_caps {
>  	uint32_t  device_id;
>  	struct ark_caps  caps;
>  };
> -#define SET_DEV_CAPS(id, rqp, vf)			\
> -	{id, {.rqpacing = rqp, .isvf = vf} }
> +#define SET_DEV_CAPS(id, vf)			\
> +	{id, {.isvf = vf} }
>  
>  static const struct ark_dev_caps
>  ark_device_caps[] = {
> -		     SET_DEV_CAPS(0x100d, true, false),
> -		     SET_DEV_CAPS(0x100e, true, false),
> -		     SET_DEV_CAPS(0x100f, true, false),
> -		     SET_DEV_CAPS(0x1010, false, false),
> -		     SET_DEV_CAPS(0x1017, true, false),
> -		     SET_DEV_CAPS(0x1018, true, false),
> -		     SET_DEV_CAPS(0x1019, true, false),
> -		     SET_DEV_CAPS(0x101a, true, false),
> -		     SET_DEV_CAPS(0x101b, true, false),
> -		     SET_DEV_CAPS(0x101c, true, true),
> -		     SET_DEV_CAPS(0x101e, false, false),
> -		     SET_DEV_CAPS(0x101f, false, false),
> +		     SET_DEV_CAPS(0x100d, false),
> +		     SET_DEV_CAPS(0x100e, false),
> +		     SET_DEV_CAPS(0x100f, false),
> +		     SET_DEV_CAPS(0x1010, false),
> +		     SET_DEV_CAPS(0x1017, false),
> +		     SET_DEV_CAPS(0x1018, false),
> +		     SET_DEV_CAPS(0x1019, false),
> +		     SET_DEV_CAPS(0x101a, false),
> +		     SET_DEV_CAPS(0x101b, false),
> +		     SET_DEV_CAPS(0x101c, true),
> +		     SET_DEV_CAPS(0x101e, false),
> +		     SET_DEV_CAPS(0x101f, false),
>  		     {.device_id = 0,}
>  };
>  
> @@ -301,9 +296,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
>  	int port_count = 1;
>  	int p;
>  	uint16_t num_queues;
> -	bool rqpacing = false;
> -
> -	ark->eth_dev = dev;
>  

Above "ark->eth_dev" assignment doesn't look directly related with RQ
pacing, I just want to double check if it is removed intentionally?

Re: [PATCH 2/3] net/ark: remove RQ pacing firmware from PMD

[Ed Czeck]

[-- Attachment #1: Type: text/plain, Size: 5037 bytes --]

On Tue, Oct 10, 2023 at 9:51 AM Ferruh Yigit <ferruh.yigit@amd.com> wrote:
>
> On 10/5/2023 9:52 PM, Ed Czeck wrote:
> > features and function have been removed from FPGA firmware
> >
>
> I am always a little confused how you manage the deployment, if a
> customer requires RQ pacing, how you manage it, at least should it be
> documented in driver documentation that RQ pacing supported before
> v23.11, or something like that?
> If this doesn't make sense for your deployment model, scratch it, this
> is just a reminder if it is useful.

Our deployment needs to balance the DPDK  release, our FPGA firmware, our
(not yet
published) DPDKpatches and external FPGA-IP firmware from AMD (Xilinx) and
Intel
(Altera).  We have safety code to ensure that these fall into a valid
alignment. We also
try to maintain SW/FPGA compatibility and evolve without breaking things
unnecessarily.
Our releases follow DPDK's and we update other tools as they are released.

For RQ pacing, it was an internal feature needed for older Xilinx PCIE IP,
with a
narrow exposure via our PMD.  The Xilinx IP no longer requires this module,
our
firmware no longer includes it, and the PMD can drop.  It was not user
controllable
nor an advertised feature.

>
>
> > Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
> > ---
> >  drivers/net/ark/ark_ethdev.c | 62 ++++++++------------------------
> >  drivers/net/ark/ark_global.h |  3 --
> >  drivers/net/ark/ark_rqp.c    | 70 ------------------------------------
> >  drivers/net/ark/ark_rqp.h    | 58 ------------------------------
> >  drivers/net/ark/meson.build  |  1 -
> >  5 files changed, 15 insertions(+), 179 deletions(-)
> >  delete mode 100644 drivers/net/ark/ark_rqp.c
> >  delete mode 100644 drivers/net/ark/ark_rqp.h
> >
> > diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
> > index 90d3c8abe6..306121ba31 100644
> > --- a/drivers/net/ark/ark_ethdev.c
> > +++ b/drivers/net/ark/ark_ethdev.c
> > @@ -17,7 +17,6 @@
> >  #include "ark_mpu.h"
> >  #include "ark_ddm.h"
> >  #include "ark_udm.h"
> > -#include "ark_rqp.h"
> >  #include "ark_pktdir.h"
> >  #include "ark_pktgen.h"
> >  #include "ark_pktchkr.h"
> > @@ -107,36 +106,32 @@ static const struct rte_pci_id pci_id_ark_map[] =
{
> >   * This structure is used to statically define the capabilities
> >   * of supported devices.
> >   * Capabilities:
> > - *  rqpacing -
> > - * Some HW variants require that PCIe read-requests be correctly
throttled.
> > - * This is called "rqpacing" and has to do with credit and flow control
> > - * on certain Arkville implementations.
> > + *    isvf -- defined for function id that are virtual
> >   */
> >  struct ark_caps {
> > -     bool rqpacing;
> >       bool isvf;
> >  };
> >  struct ark_dev_caps {
> >       uint32_t  device_id;
> >       struct ark_caps  caps;
> >  };
> > -#define SET_DEV_CAPS(id, rqp, vf)                    \
> > -     {id, {.rqpacing = rqp, .isvf = vf} }
> > +#define SET_DEV_CAPS(id, vf)                 \
> > +     {id, {.isvf = vf} }
> >
> >  static const struct ark_dev_caps
> >  ark_device_caps[] = {
> > -                  SET_DEV_CAPS(0x100d, true, false),
> > -                  SET_DEV_CAPS(0x100e, true, false),
> > -                  SET_DEV_CAPS(0x100f, true, false),
> > -                  SET_DEV_CAPS(0x1010, false, false),
> > -                  SET_DEV_CAPS(0x1017, true, false),
> > -                  SET_DEV_CAPS(0x1018, true, false),
> > -                  SET_DEV_CAPS(0x1019, true, false),
> > -                  SET_DEV_CAPS(0x101a, true, false),
> > -                  SET_DEV_CAPS(0x101b, true, false),
> > -                  SET_DEV_CAPS(0x101c, true, true),
> > -                  SET_DEV_CAPS(0x101e, false, false),
> > -                  SET_DEV_CAPS(0x101f, false, false),
> > +                  SET_DEV_CAPS(0x100d, false),
> > +                  SET_DEV_CAPS(0x100e, false),
> > +                  SET_DEV_CAPS(0x100f, false),
> > +                  SET_DEV_CAPS(0x1010, false),
> > +                  SET_DEV_CAPS(0x1017, false),
> > +                  SET_DEV_CAPS(0x1018, false),
> > +                  SET_DEV_CAPS(0x1019, false),
> > +                  SET_DEV_CAPS(0x101a, false),
> > +                  SET_DEV_CAPS(0x101b, false),
> > +                  SET_DEV_CAPS(0x101c, true),
> > +                  SET_DEV_CAPS(0x101e, false),
> > +                  SET_DEV_CAPS(0x101f, false),
> >                    {.device_id = 0,}
> >  };
> >
> > @@ -301,9 +296,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
> >       int port_count = 1;
> >       int p;
> >       uint16_t num_queues;
> > -     bool rqpacing = false;
> > -
> > -     ark->eth_dev = dev;
> >
>
> Above "ark->eth_dev" assignment doesn't look directly related with RQ
> pacing, I just want to double check if it is removed intentionally?

This change is in error.   Thanks for catching it.  New patch to follow.

>
>

[-- Attachment #2: Type: text/html, Size: 6522 bytes --]

[PATCH v2 2/3] net/ark: remove RQ pacing firmware from PMD

[Ed Czeck]

features and function have been removed from FPGA firmware

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
v2:
* restore previously incorrectly deleted line
---
 drivers/net/ark/ark_ethdev.c | 60 ++++++++-----------------------
 drivers/net/ark/ark_global.h |  3 --
 drivers/net/ark/ark_rqp.c    | 70 ------------------------------------
 drivers/net/ark/ark_rqp.h    | 58 ------------------------------
 drivers/net/ark/meson.build  |  1 -
 5 files changed, 15 insertions(+), 177 deletions(-)
 delete mode 100644 drivers/net/ark/ark_rqp.c
 delete mode 100644 drivers/net/ark/ark_rqp.h

diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 90d3c8abe6..0ffd4b9e9e 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -17,7 +17,6 @@
 #include "ark_mpu.h"
 #include "ark_ddm.h"
 #include "ark_udm.h"
-#include "ark_rqp.h"
 #include "ark_pktdir.h"
 #include "ark_pktgen.h"
 #include "ark_pktchkr.h"
@@ -107,36 +106,32 @@ static const struct rte_pci_id pci_id_ark_map[] = {
  * This structure is used to statically define the capabilities
  * of supported devices.
  * Capabilities:
- *  rqpacing -
- * Some HW variants require that PCIe read-requests be correctly throttled.
- * This is called "rqpacing" and has to do with credit and flow control
- * on certain Arkville implementations.
+ *    isvf -- defined for function id that are virtual
  */
 struct ark_caps {
-	bool rqpacing;
 	bool isvf;
 };
 struct ark_dev_caps {
 	uint32_t  device_id;
 	struct ark_caps  caps;
 };
-#define SET_DEV_CAPS(id, rqp, vf)			\
-	{id, {.rqpacing = rqp, .isvf = vf} }
+#define SET_DEV_CAPS(id, vf)			\
+	{id, {.isvf = vf} }
 
 static const struct ark_dev_caps
 ark_device_caps[] = {
-		     SET_DEV_CAPS(0x100d, true, false),
-		     SET_DEV_CAPS(0x100e, true, false),
-		     SET_DEV_CAPS(0x100f, true, false),
-		     SET_DEV_CAPS(0x1010, false, false),
-		     SET_DEV_CAPS(0x1017, true, false),
-		     SET_DEV_CAPS(0x1018, true, false),
-		     SET_DEV_CAPS(0x1019, true, false),
-		     SET_DEV_CAPS(0x101a, true, false),
-		     SET_DEV_CAPS(0x101b, true, false),
-		     SET_DEV_CAPS(0x101c, true, true),
-		     SET_DEV_CAPS(0x101e, false, false),
-		     SET_DEV_CAPS(0x101f, false, false),
+		     SET_DEV_CAPS(0x100d, false),
+		     SET_DEV_CAPS(0x100e, false),
+		     SET_DEV_CAPS(0x100f, false),
+		     SET_DEV_CAPS(0x1010, false),
+		     SET_DEV_CAPS(0x1017, false),
+		     SET_DEV_CAPS(0x1018, false),
+		     SET_DEV_CAPS(0x1019, false),
+		     SET_DEV_CAPS(0x101a, false),
+		     SET_DEV_CAPS(0x101b, false),
+		     SET_DEV_CAPS(0x101c, true),
+		     SET_DEV_CAPS(0x101e, false),
+		     SET_DEV_CAPS(0x101f, false),
 		     {.device_id = 0,}
 };
 
@@ -301,7 +296,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	int port_count = 1;
 	int p;
 	uint16_t num_queues;
-	bool rqpacing = false;
 
 	ark->eth_dev = dev;
 
@@ -319,7 +313,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	p = 0;
 	while (ark_device_caps[p].device_id != 0) {
 		if (pci_dev->id.device_id == ark_device_caps[p].device_id) {
-			rqpacing = ark_device_caps[p].caps.rqpacing;
 			ark->isvf = ark_device_caps[p].caps.isvf;
 			break;
 		}
@@ -344,12 +337,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	ark->pktgen.v  = (void *)&ark->bar0[ARK_PKTGEN_BASE];
 	ark->pktchkr.v  = (void *)&ark->bar0[ARK_PKTCHKR_BASE];
 
-	if (rqpacing) {
-		ark->rqpacing =
-			(struct ark_rqpace_t *)(ark->bar0 + ARK_RCPACING_BASE);
-	} else {
-		ark->rqpacing = NULL;
-	}
 	ark->started = 0;
 	ark->pkt_dir_v = ARK_PKT_DIR_INIT_VAL;
 
@@ -368,17 +355,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 			    ark->sysctrl.t32[4], __func__);
 		return -1;
 	}
-	if (ark->sysctrl.t32[3] != 0) {
-		if (ark->rqpacing) {
-			if (ark_rqp_lasped(ark->rqpacing)) {
-				ARK_PMD_LOG(ERR, "Arkville Evaluation System - "
-					    "Timer has Expired\n");
-				return -1;
-			}
-			ARK_PMD_LOG(WARNING, "Arkville Evaluation System - "
-				    "Timer is Running\n");
-		}
-	}
 
 	ARK_PMD_LOG(DEBUG,
 		    "HW Sanity test has PASSED, expected constant"
@@ -550,9 +526,6 @@ ark_config_device(struct rte_eth_dev *dev)
 		mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET);
 	}
 
-	if (!ark->isvf && ark->rqpacing)
-		ark_rqp_stats_reset(ark->rqpacing);
-
 	return 0;
 }
 
@@ -709,9 +682,6 @@ eth_ark_dev_close(struct rte_eth_dev *dev)
 	/*
 	 * This should only be called once for the device during shutdown
 	 */
-	if (ark->rqpacing)
-		ark_rqp_dump(ark->rqpacing);
-
 	/* return to power-on state */
 	if (ark->pd)
 		ark_pktdir_setup(ark->pd, ARK_PKT_DIR_INIT_VAL);
diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h
index 2f198edfe4..147b14b6c0 100644
--- a/drivers/net/ark/ark_global.h
+++ b/drivers/net/ark/ark_global.h
@@ -32,7 +32,6 @@
 #define ARK_CMAC_BASE     0x80000
 #define ARK_PKTDIR_BASE   0xa0000
 #define ARK_PKTCHKR_BASE  0x90000
-#define ARK_RCPACING_BASE 0xb0000
 #define ARK_EXTERNAL_BASE 0x100000
 #define ARK_MPU_QOFFSET   0x00100
 #define ARK_MAX_PORTS     RTE_MAX_ETHPORTS
@@ -150,8 +149,6 @@ struct ark_adapter {
 	int started;
 	uint16_t rx_queues;
 	uint16_t tx_queues;
-
-	struct ark_rqpace_t *rqpacing;
 };
 
 typedef uint32_t *ark_t;
diff --git a/drivers/net/ark/ark_rqp.c b/drivers/net/ark/ark_rqp.c
deleted file mode 100644
index efb9730fe6..0000000000
--- a/drivers/net/ark/ark_rqp.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
- */
-
-#include <unistd.h>
-
-#include "ark_rqp.h"
-#include "ark_logs.h"
-
-/* ************************************************************************* */
-void
-ark_rqp_stats_reset(struct ark_rqpace_t *rqp)
-{
-	rqp->stats_clear = 1;
-	/* POR 992 */
-	/* rqp->cpld_max = 992; */
-	/* POR 64 */
-	/* rqp->cplh_max = 64; */
-}
-
-/* ************************************************************************* */
-void
-ark_rqp_dump(struct ark_rqpace_t *rqp)
-{
-	if (rqp->err_count_other || rqp->cmpl_errors)
-		ARK_PMD_LOG(ERR,
-			    "RQP Errors noted: ctrl: %d cplh_hmax %d cpld_max %d"
-			    ARK_SU32
-			    ARK_SU32
-			    ARK_SU32 "\n",
-			    rqp->ctrl, rqp->cplh_max, rqp->cpld_max,
-			    "Error Count", rqp->err_cnt,
-			    "Error General", rqp->err_count_other,
-			    "Cmpl Errors", rqp->cmpl_errors);
-
-	ARK_PMD_LOG(INFO, "RQP Dump: ctrl: %d cplh_hmax %d cpld_max %d"
-		      ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 "\n",
-		      rqp->ctrl, rqp->cplh_max, rqp->cpld_max,
-		      "Error Count", rqp->err_cnt,
-		      "Error General", rqp->err_count_other,
-		      "stall_pS", rqp->stall_ps,
-		      "stall_pS Min", rqp->stall_ps_min,
-		      "stall_pS Max", rqp->stall_ps_max,
-		      "req_pS", rqp->req_ps,
-		      "req_pS Min", rqp->req_ps_min,
-		      "req_pS Max", rqp->req_ps_max,
-		      "req_dWPS", rqp->req_dw_ps,
-		      "req_dWPS Min", rqp->req_dw_ps_min,
-		      "req_dWPS Max", rqp->req_dw_ps_max,
-		      "cpl_pS", rqp->cpl_ps,
-		      "cpl_pS Min", rqp->cpl_ps_min,
-		      "cpl_pS Max", rqp->cpl_ps_max,
-		      "cpl_dWPS", rqp->cpl_dw_ps,
-		      "cpl_dWPS Min", rqp->cpl_dw_ps_min,
-		      "cpl_dWPS Max", rqp->cpl_dw_ps_max,
-		      "cplh pending", rqp->cplh_pending,
-		      "cpld pending", rqp->cpld_pending,
-		      "cplh pending max", rqp->cplh_pending_max,
-		      "cpld pending max", rqp->cpld_pending_max);
-}
-
-int
-ark_rqp_lasped(struct ark_rqpace_t *rqp)
-{
-	return rqp->lasped;
-}
diff --git a/drivers/net/ark/ark_rqp.h b/drivers/net/ark/ark_rqp.h
deleted file mode 100644
index d09f242e1e..0000000000
--- a/drivers/net/ark/ark_rqp.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
- */
-
-#ifndef _ARK_RQP_H_
-#define _ARK_RQP_H_
-
-#include <stdint.h>
-
-#include <rte_memory.h>
-
-/* The RQP or ReQuest Pacer is an internal Arkville hardware module
- * which limits the PCIE data flow to insure correct operation for the
- * particular hardware PCIE endpoint.
- * This module is *not* intended for end-user manipulation, hence
- * there is minimal documentation.
- */
-
-/*
- * RQ Pacing core hardware structure
- * This is an overlay structures to a memory mapped FPGA device.  These
- * structs will never be instantiated in ram memory
- */
-struct ark_rqpace_t {
-	volatile uint32_t ctrl;
-	volatile uint32_t stats_clear;
-	volatile uint32_t cplh_max;
-	volatile uint32_t cpld_max;
-	volatile uint32_t err_cnt;
-	volatile uint32_t stall_ps;
-	volatile uint32_t stall_ps_min;
-	volatile uint32_t stall_ps_max;
-	volatile uint32_t req_ps;
-	volatile uint32_t req_ps_min;
-	volatile uint32_t req_ps_max;
-	volatile uint32_t req_dw_ps;
-	volatile uint32_t req_dw_ps_min;
-	volatile uint32_t req_dw_ps_max;
-	volatile uint32_t cpl_ps;
-	volatile uint32_t cpl_ps_min;
-	volatile uint32_t cpl_ps_max;
-	volatile uint32_t cpl_dw_ps;
-	volatile uint32_t cpl_dw_ps_min;
-	volatile uint32_t cpl_dw_ps_max;
-	volatile uint32_t cplh_pending;
-	volatile uint32_t cpld_pending;
-	volatile uint32_t cplh_pending_max;
-	volatile uint32_t cpld_pending_max;
-	volatile uint32_t err_count_other;
-	char eval[4];
-	volatile int32_t lasped;
-	volatile uint32_t cmpl_errors;
-};
-
-void ark_rqp_dump(struct ark_rqpace_t *rqp);
-void ark_rqp_stats_reset(struct ark_rqpace_t *rqp);
-int ark_rqp_lasped(struct ark_rqpace_t *rqp);
-#endif
diff --git a/drivers/net/ark/meson.build b/drivers/net/ark/meson.build
index 8d87744c22..12b3935b85 100644
--- a/drivers/net/ark/meson.build
+++ b/drivers/net/ark/meson.build
@@ -16,6 +16,5 @@ sources = files(
         'ark_pktchkr.c',
         'ark_pktdir.c',
         'ark_pktgen.c',
-        'ark_rqp.c',
         'ark_udm.c',
 )
-- 
2.34.1

Re: [PATCH v2 2/3] net/ark: remove RQ pacing firmware from PMD

[Ferruh Yigit]

On 10/10/2023 5:08 PM, Ed Czeck wrote:
> features and function have been removed from FPGA firmware
> 
> Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
> ---
> v2:
> * restore previously incorrectly deleted line
> 

Can you please send v2 of whole set, this helps on CI checks and back
tracing via patchwork etc...

[PATCH v2 1/3] net/ark: support for single function with multiple port

[Ed Czeck]

Support the creation of multiple ports from one ark device via
the use of ark pmd extension.  I.e., one device with q queue can
seen a p ports each with q/p queues.

Add unique dev_private data for each port to manage queue assignment.

This patch repairs a latent issue uncovered during testing.
Fixes: 6799275eeea6 ("net/ark: support virtual functions")
Cc: stable@dpdk.org
Backporting is not requested.

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
 drivers/net/ark/ark_ethdev.c    | 15 ++++++++++++++-
 drivers/net/ark/ark_ethdev_rx.c |  6 +++---
 drivers/net/ark/ark_ethdev_tx.c |  2 +-
 drivers/net/ark/ark_global.h    |  3 +++
 4 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 3ddcc9b461..90d3c8abe6 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -300,6 +300,7 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	int ret;
 	int port_count = 1;
 	int p;
+	uint16_t num_queues;
 	bool rqpacing = false;
 
 	ark->eth_dev = dev;
@@ -427,6 +428,7 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 			ark->user_ext.dev_get_port_count(dev,
 				 ark->user_data[dev->data->port_id]);
 	ark->num_ports = port_count;
+	num_queues = ark_api_num_queues_per_port(ark->mpurx.v, port_count);
 
 	for (p = 0; p < port_count; p++) {
 		struct rte_eth_dev *eth_dev;
@@ -452,7 +454,18 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 		}
 
 		eth_dev->device = &pci_dev->device;
-		eth_dev->data->dev_private = ark;
+		/* Device requires new dev_private data */
+		eth_dev->data->dev_private =
+			rte_zmalloc_socket(name,
+					   sizeof(struct ark_adapter),
+					   RTE_CACHE_LINE_SIZE,
+					   rte_socket_id());
+
+		memcpy(eth_dev->data->dev_private, ark,
+		       sizeof(struct ark_adapter));
+		ark = eth_dev->data->dev_private;
+		ark->qbase = p * num_queues;
+
 		eth_dev->dev_ops = ark->eth_dev->dev_ops;
 		eth_dev->tx_pkt_burst = ark->eth_dev->tx_pkt_burst;
 		eth_dev->rx_pkt_burst = ark->eth_dev->rx_pkt_burst;
diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c
index cbc0416bc2..38bc69dff4 100644
--- a/drivers/net/ark/ark_ethdev_rx.c
+++ b/drivers/net/ark/ark_ethdev_rx.c
@@ -68,7 +68,7 @@ struct ark_rx_queue {
 static int
 eth_ark_rx_hw_setup(struct rte_eth_dev *dev,
 		    struct ark_rx_queue *queue,
-		    uint16_t rx_queue_id __rte_unused, uint16_t rx_queue_idx)
+		    uint16_t rx_queue_idx)
 {
 	rte_iova_t queue_base;
 	rte_iova_t phys_addr_q_base;
@@ -124,7 +124,7 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	uint32_t i;
 	int status;
 
-	int qidx = queue_idx;
+	int qidx = ark->qbase + queue_idx;
 
 	/* We may already be setup, free memory prior to re-allocation */
 	if (dev->data->rx_queues[queue_idx] != NULL) {
@@ -215,7 +215,7 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 	/* MPU Setup */
 	if (status == 0)
-		status = eth_ark_rx_hw_setup(dev, queue, qidx, queue_idx);
+		status = eth_ark_rx_hw_setup(dev, queue, queue_idx);
 
 	if (unlikely(status != 0)) {
 		struct rte_mbuf **mbuf;
diff --git a/drivers/net/ark/ark_ethdev_tx.c b/drivers/net/ark/ark_ethdev_tx.c
index 5940a592a2..4792754f19 100644
--- a/drivers/net/ark/ark_ethdev_tx.c
+++ b/drivers/net/ark/ark_ethdev_tx.c
@@ -229,7 +229,7 @@ eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
 	struct ark_tx_queue *queue;
 	int status;
 
-	int qidx = queue_idx;
+	int qidx = ark->qbase + queue_idx;
 
 	if (!rte_is_power_of_2(nb_desc)) {
 		ARK_PMD_LOG(ERR,
diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h
index 71d0b53e03..2f198edfe4 100644
--- a/drivers/net/ark/ark_global.h
+++ b/drivers/net/ark/ark_global.h
@@ -112,7 +112,10 @@ struct ark_adapter {
 	ark_pkt_chkr_t pc;
 	ark_pkt_dir_t pd;
 
+	/* For single function, multiple ports */
 	int num_ports;
+	uint16_t qbase;
+
 	bool isvf;
 
 	/* Packet generator/checker args */
-- 
2.34.1

[PATCH v2 2/3] net/ark: remove RQ pacing firmware from PMD

[Ed Czeck]

features and function have been removed from FPGA firmware

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
v2:
* restore previously incorrectly deleted line
---
 drivers/net/ark/ark_ethdev.c | 60 ++++++++-----------------------
 drivers/net/ark/ark_global.h |  3 --
 drivers/net/ark/ark_rqp.c    | 70 ------------------------------------
 drivers/net/ark/ark_rqp.h    | 58 ------------------------------
 drivers/net/ark/meson.build  |  1 -
 5 files changed, 15 insertions(+), 177 deletions(-)
 delete mode 100644 drivers/net/ark/ark_rqp.c
 delete mode 100644 drivers/net/ark/ark_rqp.h

diff --git a/drivers/net/ark/ark_ethdev.c b/drivers/net/ark/ark_ethdev.c
index 90d3c8abe6..0ffd4b9e9e 100644
--- a/drivers/net/ark/ark_ethdev.c
+++ b/drivers/net/ark/ark_ethdev.c
@@ -17,7 +17,6 @@
 #include "ark_mpu.h"
 #include "ark_ddm.h"
 #include "ark_udm.h"
-#include "ark_rqp.h"
 #include "ark_pktdir.h"
 #include "ark_pktgen.h"
 #include "ark_pktchkr.h"
@@ -107,36 +106,32 @@ static const struct rte_pci_id pci_id_ark_map[] = {
  * This structure is used to statically define the capabilities
  * of supported devices.
  * Capabilities:
- *  rqpacing -
- * Some HW variants require that PCIe read-requests be correctly throttled.
- * This is called "rqpacing" and has to do with credit and flow control
- * on certain Arkville implementations.
+ *    isvf -- defined for function id that are virtual
  */
 struct ark_caps {
-	bool rqpacing;
 	bool isvf;
 };
 struct ark_dev_caps {
 	uint32_t  device_id;
 	struct ark_caps  caps;
 };
-#define SET_DEV_CAPS(id, rqp, vf)			\
-	{id, {.rqpacing = rqp, .isvf = vf} }
+#define SET_DEV_CAPS(id, vf)			\
+	{id, {.isvf = vf} }
 
 static const struct ark_dev_caps
 ark_device_caps[] = {
-		     SET_DEV_CAPS(0x100d, true, false),
-		     SET_DEV_CAPS(0x100e, true, false),
-		     SET_DEV_CAPS(0x100f, true, false),
-		     SET_DEV_CAPS(0x1010, false, false),
-		     SET_DEV_CAPS(0x1017, true, false),
-		     SET_DEV_CAPS(0x1018, true, false),
-		     SET_DEV_CAPS(0x1019, true, false),
-		     SET_DEV_CAPS(0x101a, true, false),
-		     SET_DEV_CAPS(0x101b, true, false),
-		     SET_DEV_CAPS(0x101c, true, true),
-		     SET_DEV_CAPS(0x101e, false, false),
-		     SET_DEV_CAPS(0x101f, false, false),
+		     SET_DEV_CAPS(0x100d, false),
+		     SET_DEV_CAPS(0x100e, false),
+		     SET_DEV_CAPS(0x100f, false),
+		     SET_DEV_CAPS(0x1010, false),
+		     SET_DEV_CAPS(0x1017, false),
+		     SET_DEV_CAPS(0x1018, false),
+		     SET_DEV_CAPS(0x1019, false),
+		     SET_DEV_CAPS(0x101a, false),
+		     SET_DEV_CAPS(0x101b, false),
+		     SET_DEV_CAPS(0x101c, true),
+		     SET_DEV_CAPS(0x101e, false),
+		     SET_DEV_CAPS(0x101f, false),
 		     {.device_id = 0,}
 };
 
@@ -301,7 +296,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	int port_count = 1;
 	int p;
 	uint16_t num_queues;
-	bool rqpacing = false;
 
 	ark->eth_dev = dev;
 
@@ -319,7 +313,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	p = 0;
 	while (ark_device_caps[p].device_id != 0) {
 		if (pci_dev->id.device_id == ark_device_caps[p].device_id) {
-			rqpacing = ark_device_caps[p].caps.rqpacing;
 			ark->isvf = ark_device_caps[p].caps.isvf;
 			break;
 		}
@@ -344,12 +337,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 	ark->pktgen.v  = (void *)&ark->bar0[ARK_PKTGEN_BASE];
 	ark->pktchkr.v  = (void *)&ark->bar0[ARK_PKTCHKR_BASE];
 
-	if (rqpacing) {
-		ark->rqpacing =
-			(struct ark_rqpace_t *)(ark->bar0 + ARK_RCPACING_BASE);
-	} else {
-		ark->rqpacing = NULL;
-	}
 	ark->started = 0;
 	ark->pkt_dir_v = ARK_PKT_DIR_INIT_VAL;
 
@@ -368,17 +355,6 @@ eth_ark_dev_init(struct rte_eth_dev *dev)
 			    ark->sysctrl.t32[4], __func__);
 		return -1;
 	}
-	if (ark->sysctrl.t32[3] != 0) {
-		if (ark->rqpacing) {
-			if (ark_rqp_lasped(ark->rqpacing)) {
-				ARK_PMD_LOG(ERR, "Arkville Evaluation System - "
-					    "Timer has Expired\n");
-				return -1;
-			}
-			ARK_PMD_LOG(WARNING, "Arkville Evaluation System - "
-				    "Timer is Running\n");
-		}
-	}
 
 	ARK_PMD_LOG(DEBUG,
 		    "HW Sanity test has PASSED, expected constant"
@@ -550,9 +526,6 @@ ark_config_device(struct rte_eth_dev *dev)
 		mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET);
 	}
 
-	if (!ark->isvf && ark->rqpacing)
-		ark_rqp_stats_reset(ark->rqpacing);
-
 	return 0;
 }
 
@@ -709,9 +682,6 @@ eth_ark_dev_close(struct rte_eth_dev *dev)
 	/*
 	 * This should only be called once for the device during shutdown
 	 */
-	if (ark->rqpacing)
-		ark_rqp_dump(ark->rqpacing);
-
 	/* return to power-on state */
 	if (ark->pd)
 		ark_pktdir_setup(ark->pd, ARK_PKT_DIR_INIT_VAL);
diff --git a/drivers/net/ark/ark_global.h b/drivers/net/ark/ark_global.h
index 2f198edfe4..147b14b6c0 100644
--- a/drivers/net/ark/ark_global.h
+++ b/drivers/net/ark/ark_global.h
@@ -32,7 +32,6 @@
 #define ARK_CMAC_BASE     0x80000
 #define ARK_PKTDIR_BASE   0xa0000
 #define ARK_PKTCHKR_BASE  0x90000
-#define ARK_RCPACING_BASE 0xb0000
 #define ARK_EXTERNAL_BASE 0x100000
 #define ARK_MPU_QOFFSET   0x00100
 #define ARK_MAX_PORTS     RTE_MAX_ETHPORTS
@@ -150,8 +149,6 @@ struct ark_adapter {
 	int started;
 	uint16_t rx_queues;
 	uint16_t tx_queues;
-
-	struct ark_rqpace_t *rqpacing;
 };
 
 typedef uint32_t *ark_t;
diff --git a/drivers/net/ark/ark_rqp.c b/drivers/net/ark/ark_rqp.c
deleted file mode 100644
index efb9730fe6..0000000000
--- a/drivers/net/ark/ark_rqp.c
+++ /dev/null
@@ -1,70 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
- */
-
-#include <unistd.h>
-
-#include "ark_rqp.h"
-#include "ark_logs.h"
-
-/* ************************************************************************* */
-void
-ark_rqp_stats_reset(struct ark_rqpace_t *rqp)
-{
-	rqp->stats_clear = 1;
-	/* POR 992 */
-	/* rqp->cpld_max = 992; */
-	/* POR 64 */
-	/* rqp->cplh_max = 64; */
-}
-
-/* ************************************************************************* */
-void
-ark_rqp_dump(struct ark_rqpace_t *rqp)
-{
-	if (rqp->err_count_other || rqp->cmpl_errors)
-		ARK_PMD_LOG(ERR,
-			    "RQP Errors noted: ctrl: %d cplh_hmax %d cpld_max %d"
-			    ARK_SU32
-			    ARK_SU32
-			    ARK_SU32 "\n",
-			    rqp->ctrl, rqp->cplh_max, rqp->cpld_max,
-			    "Error Count", rqp->err_cnt,
-			    "Error General", rqp->err_count_other,
-			    "Cmpl Errors", rqp->cmpl_errors);
-
-	ARK_PMD_LOG(INFO, "RQP Dump: ctrl: %d cplh_hmax %d cpld_max %d"
-		      ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32
-		      ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 "\n",
-		      rqp->ctrl, rqp->cplh_max, rqp->cpld_max,
-		      "Error Count", rqp->err_cnt,
-		      "Error General", rqp->err_count_other,
-		      "stall_pS", rqp->stall_ps,
-		      "stall_pS Min", rqp->stall_ps_min,
-		      "stall_pS Max", rqp->stall_ps_max,
-		      "req_pS", rqp->req_ps,
-		      "req_pS Min", rqp->req_ps_min,
-		      "req_pS Max", rqp->req_ps_max,
-		      "req_dWPS", rqp->req_dw_ps,
-		      "req_dWPS Min", rqp->req_dw_ps_min,
-		      "req_dWPS Max", rqp->req_dw_ps_max,
-		      "cpl_pS", rqp->cpl_ps,
-		      "cpl_pS Min", rqp->cpl_ps_min,
-		      "cpl_pS Max", rqp->cpl_ps_max,
-		      "cpl_dWPS", rqp->cpl_dw_ps,
-		      "cpl_dWPS Min", rqp->cpl_dw_ps_min,
-		      "cpl_dWPS Max", rqp->cpl_dw_ps_max,
-		      "cplh pending", rqp->cplh_pending,
-		      "cpld pending", rqp->cpld_pending,
-		      "cplh pending max", rqp->cplh_pending_max,
-		      "cpld pending max", rqp->cpld_pending_max);
-}
-
-int
-ark_rqp_lasped(struct ark_rqpace_t *rqp)
-{
-	return rqp->lasped;
-}
diff --git a/drivers/net/ark/ark_rqp.h b/drivers/net/ark/ark_rqp.h
deleted file mode 100644
index d09f242e1e..0000000000
--- a/drivers/net/ark/ark_rqp.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
- */
-
-#ifndef _ARK_RQP_H_
-#define _ARK_RQP_H_
-
-#include <stdint.h>
-
-#include <rte_memory.h>
-
-/* The RQP or ReQuest Pacer is an internal Arkville hardware module
- * which limits the PCIE data flow to insure correct operation for the
- * particular hardware PCIE endpoint.
- * This module is *not* intended for end-user manipulation, hence
- * there is minimal documentation.
- */
-
-/*
- * RQ Pacing core hardware structure
- * This is an overlay structures to a memory mapped FPGA device.  These
- * structs will never be instantiated in ram memory
- */
-struct ark_rqpace_t {
-	volatile uint32_t ctrl;
-	volatile uint32_t stats_clear;
-	volatile uint32_t cplh_max;
-	volatile uint32_t cpld_max;
-	volatile uint32_t err_cnt;
-	volatile uint32_t stall_ps;
-	volatile uint32_t stall_ps_min;
-	volatile uint32_t stall_ps_max;
-	volatile uint32_t req_ps;
-	volatile uint32_t req_ps_min;
-	volatile uint32_t req_ps_max;
-	volatile uint32_t req_dw_ps;
-	volatile uint32_t req_dw_ps_min;
-	volatile uint32_t req_dw_ps_max;
-	volatile uint32_t cpl_ps;
-	volatile uint32_t cpl_ps_min;
-	volatile uint32_t cpl_ps_max;
-	volatile uint32_t cpl_dw_ps;
-	volatile uint32_t cpl_dw_ps_min;
-	volatile uint32_t cpl_dw_ps_max;
-	volatile uint32_t cplh_pending;
-	volatile uint32_t cpld_pending;
-	volatile uint32_t cplh_pending_max;
-	volatile uint32_t cpld_pending_max;
-	volatile uint32_t err_count_other;
-	char eval[4];
-	volatile int32_t lasped;
-	volatile uint32_t cmpl_errors;
-};
-
-void ark_rqp_dump(struct ark_rqpace_t *rqp);
-void ark_rqp_stats_reset(struct ark_rqpace_t *rqp);
-int ark_rqp_lasped(struct ark_rqpace_t *rqp);
-#endif
diff --git a/drivers/net/ark/meson.build b/drivers/net/ark/meson.build
index 8d87744c22..12b3935b85 100644
--- a/drivers/net/ark/meson.build
+++ b/drivers/net/ark/meson.build
@@ -16,6 +16,5 @@ sources = files(
         'ark_pktchkr.c',
         'ark_pktdir.c',
         'ark_pktgen.c',
-        'ark_rqp.c',
         'ark_udm.c',
 )
-- 
2.34.1

[PATCH v2 3/3] net/ark: support for large dataroom in FPGA

[Ed Czeck]

Allow for non-standard dataroom sizes for upstream data
motion when using a custom mbuf pool.

- New firmware version for UDM (Upstream Data Mover)
- New layout on ark_rx_meta data

Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
---
 doc/guides/nics/ark.rst         | 1 +
 drivers/net/ark/ark_ethdev_rx.c | 8 ++++++++
 drivers/net/ark/ark_ext.h       | 9 +++++++++
 drivers/net/ark/ark_udm.h       | 5 +++--
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/doc/guides/nics/ark.rst b/doc/guides/nics/ark.rst
index e1683cf782..bcc9f505df 100644
--- a/doc/guides/nics/ark.rst
+++ b/doc/guides/nics/ark.rst
@@ -334,6 +334,7 @@ with Arkville releases 21.05, 21.08 and 21.11. LTS versions of DPDK remain
 compatible with the corresponding Arkville version.  If other combinations
 are required, please contact Atomic Rules support.
 
+* DPDK 23.11 requires Arkville 23.11.
 * DPDK 22.07 requires Arkville 22.07.
 * DPDK 22.03 requires Arkville 22.03.
 * DPDK 21.05 requires Arkville 21.05.
diff --git a/drivers/net/ark/ark_ethdev_rx.c b/drivers/net/ark/ark_ethdev_rx.c
index 38bc69dff4..24f1c65690 100644
--- a/drivers/net/ark/ark_ethdev_rx.c
+++ b/drivers/net/ark/ark_ethdev_rx.c
@@ -9,6 +9,7 @@
 #include "ark_logs.h"
 #include "ark_mpu.h"
 #include "ark_udm.h"
+#include "ark_ext.h"
 
 #define ARK_RX_META_SIZE 32
 #define ARK_RX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_RX_META_SIZE)
@@ -166,6 +167,13 @@ eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	queue->mb_pool = mb_pool;
 	queue->dataroom = rte_pktmbuf_data_room_size(mb_pool) -
 		RTE_PKTMBUF_HEADROOM;
+
+	/* Check pool's private data to confirm pool structure */
+	if (mb_pool->private_data_size != 0) {
+		struct rte_pmd_ark_lmbuf_mempool_priv *pool_priv = rte_mempool_get_priv(mb_pool);
+		if (strncmp(pool_priv->cookie, ARK_MEMPOOL_COOKIE, sizeof(pool_priv->cookie)) == 0)
+			queue->dataroom = pool_priv->dataroom;
+	}
 	queue->headroom = RTE_PKTMBUF_HEADROOM;
 	queue->phys_qid = qidx;
 	queue->queue_index = queue_idx;
diff --git a/drivers/net/ark/ark_ext.h b/drivers/net/ark/ark_ext.h
index d235d0ff85..6d37449195 100644
--- a/drivers/net/ark/ark_ext.h
+++ b/drivers/net/ark/ark_ext.h
@@ -5,6 +5,7 @@
 #ifndef _ARK_EXT_H_
 #define _ARK_EXT_H_
 
+#include <rte_mbuf.h>
 #include <ethdev_driver.h>
 
 /* The following section lists function prototypes for Arkville's
@@ -16,6 +17,14 @@
  * See documentation for compiling and use of extensions.
  */
 
+/* private data optionally attached to mempool for rx */
+struct rte_pmd_ark_lmbuf_mempool_priv {
+	struct rte_pktmbuf_pool_private pool_private;
+	char cookie[4];
+	uint32_t dataroom;
+};
+#define ARK_MEMPOOL_COOKIE "ARK1"
+
 /**
  * Extension prototype, required implementation if extensions are used.
  * Called during device probe to initialize the user structure
diff --git a/drivers/net/ark/ark_udm.h b/drivers/net/ark/ark_udm.h
index f0685c95c7..ec53ec7e79 100644
--- a/drivers/net/ark/ark_udm.h
+++ b/drivers/net/ark/ark_udm.h
@@ -20,7 +20,8 @@
  */
 struct ark_rx_meta {
 	uint32_t user_meta[5];	/* user defined based on fpga code */
-	uint8_t  reserved[10];
+	uint32_t pkt_len32;
+	uint8_t  reserved[6];
 	uint16_t pkt_len;
 } __rte_packed;
 
@@ -33,7 +34,7 @@ struct ark_rx_meta {
 #define ARK_RX_WRITE_TIME_NS 2500
 #define ARK_UDM_SETUP 0
 #define ARK_UDM_MODID 0x4d445500
-#define ARK_UDM_MODVER 0x37313232
+#define ARK_UDM_MODVER 0x37333332
 
 struct ark_udm_setup_t {
 	union {
-- 
2.34.1

Re: [PATCH v2 1/3] net/ark: support for single function with multiple port

[Ferruh Yigit]

On 10/10/2023 9:42 PM, Ed Czeck wrote:
> Support the creation of multiple ports from one ark device via
> the use of ark pmd extension.  I.e., one device with q queue can
> seen a p ports each with q/p queues.
> 
> Add unique dev_private data for each port to manage queue assignment.
> 
> This patch repairs a latent issue uncovered during testing.
> Fixes: 6799275eeea6 ("net/ark: support virtual functions")
> Cc: stable@dpdk.org
> Backporting is not requested.
> 
> Signed-off-by: Ed Czeck <ed.czeck@atomicrules.com>
> 

Series applied to dpdk-next-net/main, thanks.