[PATCH v1 21/31] net/ntnic: add base init and deinit of the NT flow API

[Serhii Iliushyk <sil-plv@napatech.com>]

From: Oleksandr Kolomeiets <okl-plv@napatech.com>

Add basic API for initialization resources required by flow filter API

Signed-off-by: Oleksandr Kolomeiets <okl-plv@napatech.com>
---
 drivers/net/ntnic/include/flow_api.h          |  82 ++++++
 drivers/net/ntnic/include/flow_api_engine.h   |  43 +++
 drivers/net/ntnic/include/hw_mod_backend.h    |   3 +
 .../ntnic/include/stream_binary_flow_api.h    |  22 ++
 drivers/net/ntnic/meson.build                 |   2 +
 drivers/net/ntnic/nthw/flow_api/flow_api.c    | 245 ++++++++++++++++++
 .../ntnic/nthw/flow_api/flow_api_nic_setup.h  |   6 +
 drivers/net/ntnic/nthw/flow_api/flow_filter.c |  17 +-
 drivers/net/ntnic/nthw/flow_api/flow_kcc.c    |  19 ++
 drivers/net/ntnic/nthw/flow_api/flow_km.c     |  19 ++
 10 files changed, 453 insertions(+), 5 deletions(-)
 create mode 100644 drivers/net/ntnic/include/flow_api_engine.h
 create mode 100644 drivers/net/ntnic/include/stream_binary_flow_api.h
 create mode 100644 drivers/net/ntnic/nthw/flow_api/flow_kcc.c
 create mode 100644 drivers/net/ntnic/nthw/flow_api/flow_km.c

diff --git a/drivers/net/ntnic/include/flow_api.h b/drivers/net/ntnic/include/flow_api.h
index 6a2277c2ca..bad1f72868 100644
--- a/drivers/net/ntnic/include/flow_api.h
+++ b/drivers/net/ntnic/include/flow_api.h
@@ -6,14 +6,96 @@
 #ifndef _FLOW_API_H_
 #define _FLOW_API_H_
 
+#include <pthread.h>
+
 #include "ntlog.h"
 
+#include "flow_api_engine.h"
 #include "hw_mod_backend.h"
+#include "stream_binary_flow_api.h"
+
+/*
+ * Flow NIC and Eth port device management
+ */
+
+struct hw_mod_resource_s {
+	uint8_t *alloc_bm;      /* allocation bitmap */
+	uint32_t *ref;  /* reference counter for each resource element */
+	uint32_t resource_count;/* number of total available entries */
+};
+
+/*
+ * Device Management API
+ */
+int flow_delete_eth_dev(struct flow_eth_dev *eth_dev);
+
+struct flow_eth_dev {
+	/* NIC that owns this port device */
+	struct flow_nic_dev *ndev;
+	/* NIC port id */
+	uint8_t port;
+
+	/* 0th for exception */
+	struct flow_queue_id_s rx_queue[FLOW_MAX_QUEUES + 1];
+
+	/* VSWITCH has exceptions sent on queue 0 per design */
+	int num_queues;
+
+	struct flow_eth_dev *next;
+};
 
 /* registered NIC backends */
 struct flow_nic_dev {
+	struct hw_mod_resource_s res[RES_COUNT];/* raw NIC resource allocation table */
+	void *km_res_handle;
+	void *kcc_res_handle;
+
+	uint32_t flow_unique_id_counter;
+	/* linked list of all flows created on this NIC */
+	struct flow_handle *flow_base;
+
 	/* NIC backend API */
 	struct flow_api_backend_s be;
+	/* linked list of created eth-port devices on this NIC */
+	struct flow_eth_dev *eth_base;
+	pthread_mutex_t mtx;
+
+	/* next NIC linked list */
+	struct flow_nic_dev *next;
 };
 
+/*
+ * Resources
+ */
+
+extern const char *dbg_res_descr[];
+
+#define flow_nic_unset_bit(arr, x)                                                                \
+	do {                                                                                      \
+		size_t _temp_x = (x);                                                             \
+		arr[_temp_x / 8] &= (uint8_t)(~(1 << (_temp_x % 8)));                             \
+	} while (0)
+
+#define flow_nic_is_bit_set(arr, x)                                                               \
+	({                                                                                        \
+		size_t _temp_x = (x);                                                             \
+		(arr[_temp_x / 8] & (uint8_t)(1 << (_temp_x % 8)));                               \
+	})
+
+#define flow_nic_mark_resource_unused(_ndev, res_type, index)                                     \
+	do {                                                                                      \
+		typeof(res_type) _temp_res_type = (res_type);                                 \
+		size_t _temp_index = (index);                                                     \
+		NT_LOG(DBG, FILTER, "mark resource unused: %s idx %zu\n",                         \
+		       dbg_res_descr[_temp_res_type], _temp_index);                               \
+		flow_nic_unset_bit((_ndev)->res[_temp_res_type].alloc_bm, _temp_index);           \
+	} while (0)
+
+#define flow_nic_is_resource_used(_ndev, res_type, index)                                         \
+	(!!flow_nic_is_bit_set((_ndev)->res[res_type].alloc_bm, index))
+
+void flow_nic_free_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int idx);
+
+int flow_nic_deref_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int index);
+
 #endif
diff --git a/drivers/net/ntnic/include/flow_api_engine.h b/drivers/net/ntnic/include/flow_api_engine.h
new file mode 100644
index 0000000000..724b68c3e8
--- /dev/null
+++ b/drivers/net/ntnic/include/flow_api_engine.h
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#ifndef _FLOW_API_ENGINE_H_
+#define _FLOW_API_ENGINE_H_
+
+/*
+ * Resource management
+ * These are free resources in FPGA
+ * Other FPGA memory lists are linked to one of these
+ * and will implicitly follow them
+ */
+enum res_type_e {
+	RES_QUEUE,
+	RES_CAT_CFN,
+	RES_CAT_COT,
+	RES_CAT_EXO,
+	RES_CAT_LEN,
+	RES_KM_FLOW_TYPE,
+	RES_KM_CATEGORY,
+	RES_HSH_RCP,
+	RES_PDB_RCP,
+	RES_QSL_RCP,
+	RES_QSL_QST,
+	RES_SLC_LR_RCP,
+
+	RES_FLM_FLOW_TYPE,
+	RES_FLM_RCP,
+	RES_TPE_RCP,
+	RES_TPE_EXT,
+	RES_TPE_RPL,
+	RES_SCRUB_RCP,
+	RES_COUNT,
+	RES_INVALID
+};
+
+void km_free_ndev_resource_management(void **handle);
+
+void kcc_free_ndev_resource_management(void **handle);
+
+#endif  /* _FLOW_API_ENGINE_H_ */
diff --git a/drivers/net/ntnic/include/hw_mod_backend.h b/drivers/net/ntnic/include/hw_mod_backend.h
index 29c33306d1..3c6c15c896 100644
--- a/drivers/net/ntnic/include/hw_mod_backend.h
+++ b/drivers/net/ntnic/include/hw_mod_backend.h
@@ -8,6 +8,8 @@
 
 #include <stdbool.h>
 
+#include "ntlog.h"
+
 #include "hw_mod_cat_v18.h"
 #include "hw_mod_cat_v21.h"
 #include "hw_mod_flm_v25.h"
@@ -275,6 +277,7 @@ struct flow_api_backend_ops {
 
 struct flow_api_backend_s {
 	void *be_dev;
+	const struct flow_api_backend_ops *iface;
 };
 
 #endif  /* _HW_MOD_BACKEND_H_ */
diff --git a/drivers/net/ntnic/include/stream_binary_flow_api.h b/drivers/net/ntnic/include/stream_binary_flow_api.h
new file mode 100644
index 0000000000..10529b8843
--- /dev/null
+++ b/drivers/net/ntnic/include/stream_binary_flow_api.h
@@ -0,0 +1,22 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#ifndef _STREAM_BINARY_FLOW_API_H_
+#define _STREAM_BINARY_FLOW_API_H_
+
+/*
+ * Flow frontend for binary programming interface
+ */
+
+#define FLOW_MAX_QUEUES 128
+
+struct flow_queue_id_s {
+	int id;
+	int hw_id;
+};
+
+struct flow_eth_dev;             /* port device */
+
+#endif  /* _STREAM_BINARY_FLOW_API_H_ */
diff --git a/drivers/net/ntnic/meson.build b/drivers/net/ntnic/meson.build
index e236b82b36..15549e1c94 100644
--- a/drivers/net/ntnic/meson.build
+++ b/drivers/net/ntnic/meson.build
@@ -46,6 +46,8 @@ sources = files(
         'nthw/flow_api/flow_api.c',
         'nthw/flow_api/flow_backend/flow_backend.c',
         'nthw/flow_api/flow_filter.c',
+        'nthw/flow_api/flow_kcc.c',
+        'nthw/flow_api/flow_km.c',
         'nthw/flow_filter/flow_nthw_cat.c',
         'nthw/flow_filter/flow_nthw_csu.c',
         'nthw/flow_filter/flow_nthw_flm.c',
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_api.c b/drivers/net/ntnic/nthw/flow_api/flow_api.c
index 9671a20e0b..92b8d083e0 100644
--- a/drivers/net/ntnic/nthw/flow_api/flow_api.c
+++ b/drivers/net/ntnic/nthw/flow_api/flow_api.c
@@ -3,11 +3,256 @@
  * Copyright(c) 2023 Napatech A/S
  */
 
+#include "flow_api_engine.h"
 #include "flow_api_nic_setup.h"
 #include "ntnic_mod_reg.h"
 
 #include "flow_filter.h"
 
+const char *dbg_res_descr[] = {
+	/* RES_QUEUE */ "RES_QUEUE",
+	/* RES_CAT_CFN */ "RES_CAT_CFN",
+	/* RES_CAT_COT */ "RES_CAT_COT",
+	/* RES_CAT_EXO */ "RES_CAT_EXO",
+	/* RES_CAT_LEN */ "RES_CAT_LEN",
+	/* RES_KM_FLOW_TYPE */ "RES_KM_FLOW_TYPE",
+	/* RES_KM_CATEGORY */ "RES_KM_CATEGORY",
+	/* RES_HSH_RCP */ "RES_HSH_RCP",
+	/* RES_PDB_RCP */ "RES_PDB_RCP",
+	/* RES_QSL_RCP */ "RES_QSL_RCP",
+	/* RES_QSL_LTX */ "RES_QSL_LTX",
+	/* RES_QSL_QST */ "RES_QSL_QST",
+	/* RES_SLC_LR_RCP */ "RES_SLC_LR_RCP",
+	/* RES_FLM_FLOW_TYPE */ "RES_FLM_FLOW_TYPE",
+	/* RES_FLM_RCP */ "RES_FLM_RCP",
+	/* RES_TPE_RCP */ "RES_TPE_RCP",
+	/* RES_TPE_EXT */ "RES_TPE_EXT",
+	/* RES_TPE_RPL */ "RES_TPE_RPL",
+	/* RES_COUNT */ "RES_COUNT",
+	/* RES_INVALID */ "RES_INVALID"
+};
+
+static struct flow_nic_dev *dev_base;
+static pthread_mutex_t base_mtx = PTHREAD_MUTEX_INITIALIZER;
+
+void flow_nic_free_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int idx)
+{
+	flow_nic_mark_resource_unused(ndev, res_type, idx);
+}
+
+int flow_nic_deref_resource(struct flow_nic_dev *ndev, enum res_type_e res_type, int index)
+{
+	NT_LOG(DBG, FILTER, "De-reference resource %s idx %i (before ref cnt %i)\n",
+		dbg_res_descr[res_type], index, ndev->res[res_type].ref[index]);
+	assert(flow_nic_is_resource_used(ndev, res_type, index));
+	assert(ndev->res[res_type].ref[index]);
+	/* deref */
+	ndev->res[res_type].ref[index]--;
+
+	if (!ndev->res[res_type].ref[index])
+		flow_nic_free_resource(ndev, res_type, index);
+
+	return !!ndev->res[res_type].ref[index];/* if 0 resource has been freed */
+}
+
+/*
+ * Device Management API
+ */
+
+static int nic_remove_eth_port_dev(struct flow_nic_dev *ndev, struct flow_eth_dev *eth_dev)
+{
+	struct flow_eth_dev *dev = ndev->eth_base, *prev = NULL;
+
+	while (dev) {
+		if (dev == eth_dev) {
+			if (prev)
+				prev->next = dev->next;
+
+			else
+				ndev->eth_base = dev->next;
+
+			return 0;
+		}
+
+		prev = dev;
+		dev = dev->next;
+	}
+
+	return -1;
+}
+
+static void flow_ndev_reset(struct flow_nic_dev *ndev)
+{
+	/* Delete all eth-port devices created on this NIC device */
+	while (ndev->eth_base)
+		flow_delete_eth_dev(ndev->eth_base);
+
+	km_free_ndev_resource_management(&ndev->km_res_handle);
+	kcc_free_ndev_resource_management(&ndev->kcc_res_handle);
+
+	ndev->flow_unique_id_counter = 0;
+
+#ifdef FLOW_DEBUG
+	/*
+	 * free all resources default allocated, initially for this NIC DEV
+	 * Is not really needed since the bitmap will be freed in a sec. Therefore
+	 * only in debug mode
+	 */
+
+	/* Check if all resources has been released */
+	NT_LOG(DBG, FILTER, "Delete NIC DEV Adaptor %i\n", ndev->adapter_no);
+
+	for (unsigned int i = 0; i < RES_COUNT; i++) {
+		int err = 0;
+#if defined(FLOW_DEBUG)
+		NT_LOG(DBG, FILTER, "RES state for: %s\n", dbg_res_descr[i]);
+#endif
+
+		for (unsigned int ii = 0; ii < ndev->res[i].resource_count; ii++) {
+			int ref = ndev->res[i].ref[ii];
+			int used = flow_nic_is_resource_used(ndev, i, ii);
+
+			if (ref || used) {
+				NT_LOG(DBG, FILTER, "  [%i]: ref cnt %i, used %i\n", ii, ref,
+					used);
+				err = 1;
+			}
+		}
+
+		if (err)
+			NT_LOG(DBG, FILTER, "ERROR - some resources not freed\n");
+	}
+
+#endif
+}
+
+int flow_delete_eth_dev(struct flow_eth_dev *eth_dev)
+{
+	struct flow_nic_dev *ndev = eth_dev->ndev;
+
+	if (!ndev) {
+		/* Error invalid nic device */
+		return -1;
+	}
+
+	NT_LOG(DBG, FILTER, "Delete eth-port device %p, port %i\n", eth_dev, eth_dev->port);
+
+#ifdef FLOW_DEBUG
+	ndev->be.iface->set_debug_mode(ndev->be.be_dev, FLOW_BACKEND_DEBUG_MODE_WRITE);
+#endif
+
+	/* delete all created flows from this device */
+	pthread_mutex_lock(&ndev->mtx);
+
+#ifdef FLOW_DEBUG
+	ndev->be.iface->set_debug_mode(ndev->be.be_dev, FLOW_BACKEND_DEBUG_MODE_NONE);
+#endif
+
+#ifndef SCATTER_GATHER
+
+	/* free rx queues */
+	for (int i = 0; i < eth_dev->num_queues; i++) {
+		ndev->be.iface->free_rx_queue(ndev->be.be_dev, eth_dev->rx_queue[i].hw_id);
+		flow_nic_deref_resource(ndev, RES_QUEUE, eth_dev->rx_queue[i].id);
+	}
+
+#endif
+
+	/* take eth_dev out of ndev list */
+	if (nic_remove_eth_port_dev(ndev, eth_dev) != 0)
+		NT_LOG(ERR, FILTER, "ERROR : eth_dev %p not found\n", eth_dev);
+
+	pthread_mutex_unlock(&ndev->mtx);
+
+	/* free eth_dev */
+	free(eth_dev);
+
+	return 0;
+}
+
+/*
+ * Flow API NIC Setup
+ * Flow backend creation function - register and initialize common backend API to FPA modules
+ */
+
+static void done_resource_elements(struct flow_nic_dev *ndev, enum res_type_e res_type)
+{
+	assert(ndev);
+
+	if (ndev->res[res_type].alloc_bm)
+		free(ndev->res[res_type].alloc_bm);
+}
+
+static int list_remove_flow_nic(struct flow_nic_dev *ndev)
+{
+	pthread_mutex_lock(&base_mtx);
+	struct flow_nic_dev *nic_dev = dev_base, *prev = NULL;
+
+	while (nic_dev) {
+		if (nic_dev == ndev) {
+			if (prev)
+				prev->next = nic_dev->next;
+
+			else
+				dev_base = nic_dev->next;
+
+			pthread_mutex_unlock(&base_mtx);
+			return 0;
+		}
+
+		prev = nic_dev;
+		nic_dev = nic_dev->next;
+	}
+
+	pthread_mutex_unlock(&base_mtx);
+	return -1;
+}
+
+struct flow_nic_dev *flow_api_create(uint8_t adapter_no, const struct flow_api_backend_ops *be_if,
+	void *be_dev)
+{
+	(void)adapter_no;
+
+	if (!be_if || be_if->version != 1) {
+		NT_LOG(DBG, FILTER, "ERR: %s\n", __func__);
+		return NULL;
+	}
+
+	struct flow_nic_dev *ndev = calloc(1, sizeof(struct flow_nic_dev));
+
+	if (!ndev) {
+		NT_LOG(ERR, FILTER, "ERROR: calloc failed\n");
+		return NULL;
+	}
+
+	/*
+	 * To dump module initialization writes use
+	 * FLOW_BACKEND_DEBUG_MODE_WRITE
+	 * then remember to set it ...NONE afterwards again
+	 */
+	be_if->set_debug_mode(be_dev, FLOW_BACKEND_DEBUG_MODE_NONE);
+
+	return ndev;
+}
+
+int flow_api_done(struct flow_nic_dev *ndev)
+{
+	NT_LOG(DBG, FILTER, "FLOW API DONE\n");
+
+	if (ndev) {
+		flow_ndev_reset(ndev);
+
+		/* delete resource management allocations for this ndev */
+		for (int i = 0; i < RES_COUNT; i++)
+			done_resource_elements(ndev, i);
+
+		list_remove_flow_nic(ndev);
+		free(ndev);
+	}
+
+	return 0;
+}
+
 void *flow_api_get_be_dev(struct flow_nic_dev *ndev)
 {
 	if (!ndev) {
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_api_nic_setup.h b/drivers/net/ntnic/nthw/flow_api/flow_api_nic_setup.h
index da083f050a..eeb070c569 100644
--- a/drivers/net/ntnic/nthw/flow_api/flow_api_nic_setup.h
+++ b/drivers/net/ntnic/nthw/flow_api/flow_api_nic_setup.h
@@ -9,6 +9,12 @@
 #include "hw_mod_backend.h"
 #include "flow_api.h"
 
+/*
+ * Flow capable NIC backend - creating flow api instance for adapter nr (backend)
+ */
+struct flow_nic_dev *flow_api_create(uint8_t adapter_no, const struct flow_api_backend_ops *be_if,
+	void *be_dev);
+int flow_api_done(struct flow_nic_dev *dev);
 void *flow_api_get_be_dev(struct flow_nic_dev *dev);
 
 #endif  /* __FLOW_API_NIC_SETUP_H__ */
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_filter.c b/drivers/net/ntnic/nthw/flow_api/flow_filter.c
index e822ba7df9..69e5972f5d 100644
--- a/drivers/net/ntnic/nthw/flow_api/flow_filter.c
+++ b/drivers/net/ntnic/nthw/flow_api/flow_filter.c
@@ -9,10 +9,8 @@
 
 int flow_filter_init(nthw_fpga_t *p_fpga, struct flow_nic_dev **p_flow_device, int adapter_no)
 {
-	(void)p_flow_device;
-	(void)adapter_no;
-
 	void *be_dev = NULL;
+	struct flow_nic_dev *flow_nic;
 
 	const struct flow_backend_ops *flow_backend_ops = get_flow_backend_ops();
 
@@ -22,8 +20,17 @@ int flow_filter_init(nthw_fpga_t *p_fpga, struct flow_nic_dev **p_flow_device, i
 	}
 
 	NT_LOG(DBG, FILTER, "Initializing flow filter api\n");
-	flow_backend_ops->bin_flow_backend_init(p_fpga, &be_dev);
+	const struct flow_api_backend_ops *iface =
+		flow_backend_ops->bin_flow_backend_init(p_fpga, &be_dev);
+
+	flow_nic = flow_api_create((uint8_t)adapter_no, iface, be_dev);
+
+	if (!flow_nic) {
+		*p_flow_device = NULL;
+		return -1;
+	}
 
+	*p_flow_device = flow_nic;
 	return 0;
 }
 
@@ -31,7 +38,7 @@ int flow_filter_done(struct flow_nic_dev *dev)
 {
 	void *be_dev = flow_api_get_be_dev(dev);
 
-	int res = 0;
+	int res = flow_api_done(dev);
 
 	if (be_dev) {
 		const struct flow_backend_ops *flow_backend_ops = get_flow_backend_ops();
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_kcc.c b/drivers/net/ntnic/nthw/flow_api/flow_kcc.c
new file mode 100644
index 0000000000..dc5e5474c1
--- /dev/null
+++ b/drivers/net/ntnic/nthw/flow_api/flow_kcc.c
@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#include <stdlib.h>
+
+#include "hw_mod_backend.h"
+#include "flow_api_engine.h"
+
+void kcc_free_ndev_resource_management(void **handle)
+{
+	if (*handle) {
+		free(*handle);
+		NT_LOG(DBG, FILTER, "Free NIC DEV KCC-CAM record manager\n");
+	}
+
+	*handle = NULL;
+}
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_km.c b/drivers/net/ntnic/nthw/flow_api/flow_km.c
new file mode 100644
index 0000000000..1ba7ed7da2
--- /dev/null
+++ b/drivers/net/ntnic/nthw/flow_api/flow_km.c
@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Napatech A/S
+ */
+
+#include <stdlib.h>
+
+#include "hw_mod_backend.h"
+#include "flow_api_engine.h"
+
+void km_free_ndev_resource_management(void **handle)
+{
+	if (*handle) {
+		free(*handle);
+		NT_LOG(DBG, FILTER, "Free NIC DEV CAM and TCAM record manager\n");
+	}
+
+	*handle = NULL;
+}
-- 
2.45.0