[PATCH v5 78/80] net/ntnic: add async flow create/delete implementation

[Serhii Iliushyk <sil-plv@napatech.com>]

From: Danylo Vodopianov <dvo-plv@napatech.com>

Inline profile was extended with async flow create and delete features
implementation.

async create and destroy was added to the flow filter ops.

Signed-off-by: Danylo Vodopianov <dvo-plv@napatech.com>
---
 doc/guides/nics/ntnic.rst                     |   1 +
 doc/guides/rel_notes/release_24_11.rst        |   1 +
 drivers/net/ntnic/include/flow_api_engine.h   |  36 +++
 drivers/net/ntnic/nthw/flow_api/flow_api.c    |  39 +++
 .../profile_inline/flow_api_hw_db_inline.c    |  13 +
 .../profile_inline/flow_api_hw_db_inline.h    |   2 +
 .../profile_inline/flow_api_profile_inline.c  | 248 +++++++++++++++++-
 .../profile_inline/flow_api_profile_inline.h  |  14 +
 drivers/net/ntnic/ntnic_mod_reg.h             |  15 ++
 9 files changed, 368 insertions(+), 1 deletion(-)

diff --git a/doc/guides/nics/ntnic.rst b/doc/guides/nics/ntnic.rst
index afdaf22e0b..fa6cd2b95c 100644
--- a/doc/guides/nics/ntnic.rst
+++ b/doc/guides/nics/ntnic.rst
@@ -70,6 +70,7 @@ Features
 - Flow aging support
 - Flow metering, including meter policy API.
 - Flow update. Update of the action list for specific flow
+- Asynchronous flow support
 
 Limitations
 ~~~~~~~~~~~
diff --git a/doc/guides/rel_notes/release_24_11.rst b/doc/guides/rel_notes/release_24_11.rst
index 735a295f6e..13f7dada4b 100644
--- a/doc/guides/rel_notes/release_24_11.rst
+++ b/doc/guides/rel_notes/release_24_11.rst
@@ -167,6 +167,7 @@ New Features
   * Added age rte flow action support
   * Added meter flow metering and flow policy support
   * Added flow actions update support
+  * Added asynchronous flow support
 
 * **Added cryptodev queue pair reset support.**
 
diff --git a/drivers/net/ntnic/include/flow_api_engine.h b/drivers/net/ntnic/include/flow_api_engine.h
index 505fb8e501..6935ff483a 100644
--- a/drivers/net/ntnic/include/flow_api_engine.h
+++ b/drivers/net/ntnic/include/flow_api_engine.h
@@ -339,6 +339,12 @@ struct flow_handle {
 			uint8_t flm_rqi;
 			uint8_t flm_qfi;
 			uint8_t flm_scrub_prof;
+
+			/* Flow specific pointer to application template table cell stored during
+			 * flow create.
+			 */
+			struct flow_template_table_cell *template_table_cell;
+			bool flm_async;
 		};
 	};
 };
@@ -347,8 +353,38 @@ struct flow_pattern_template {
 };
 
 struct flow_actions_template {
+	struct nic_flow_def *fd;
+
+	uint32_t num_dest_port;
+	uint32_t num_queues;
 };
+
+struct flow_template_table_cell {
+	atomic_int status;
+	atomic_int counter;
+
+	uint32_t flm_db_idx_counter;
+	uint32_t flm_db_idxs[RES_COUNT];
+
+	uint32_t flm_key_id;
+	uint32_t flm_ft;
+
+	uint16_t flm_rpl_ext_ptr;
+	uint8_t  flm_scrub_prof;
+};
+
 struct flow_template_table {
+	struct flow_pattern_template **pattern_templates;
+	uint8_t nb_pattern_templates;
+
+	struct flow_actions_template **actions_templates;
+	uint8_t nb_actions_templates;
+
+	struct flow_template_table_cell *pattern_action_pairs;
+
+	struct rte_flow_attr attr;
+	uint16_t forced_vlan_vid;
+	uint16_t caller_id;
 };
 
 void km_attach_ndev_resource_management(struct km_flow_def_s *km, void **handle);
diff --git a/drivers/net/ntnic/nthw/flow_api/flow_api.c b/drivers/net/ntnic/nthw/flow_api/flow_api.c
index ef8caefd9a..884a59a5de 100644
--- a/drivers/net/ntnic/nthw/flow_api/flow_api.c
+++ b/drivers/net/ntnic/nthw/flow_api/flow_api.c
@@ -1077,6 +1077,43 @@ static int flow_configure(struct flow_eth_dev *dev, uint8_t caller_id,
 			nb_queue, queue_attr, error);
 }
 
+/*
+ * Flow Asynchronous operation API
+ */
+
+static struct flow_handle *
+flow_async_create(struct flow_eth_dev *dev, uint32_t queue_id,
+	const struct rte_flow_op_attr *op_attr, struct flow_template_table *template_table,
+	const struct rte_flow_item pattern[], uint8_t pattern_template_index,
+	const struct rte_flow_action actions[], uint8_t actions_template_index, void *user_data,
+	struct rte_flow_error *error)
+{
+	const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops();
+
+	if (profile_inline_ops == NULL) {
+		NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized");
+		return NULL;
+	}
+
+	return profile_inline_ops->flow_async_create_profile_inline(dev, queue_id, op_attr,
+			template_table, pattern, pattern_template_index, actions,
+			actions_template_index, user_data, error);
+}
+
+static int flow_async_destroy(struct flow_eth_dev *dev, uint32_t queue_id,
+	const struct rte_flow_op_attr *op_attr, struct flow_handle *flow,
+	void *user_data, struct rte_flow_error *error)
+{
+	const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops();
+
+	if (profile_inline_ops == NULL) {
+		NT_LOG_DBGX(ERR, FILTER, "profile_inline module uninitialized");
+		return -1;
+	}
+
+	return profile_inline_ops->flow_async_destroy_profile_inline(dev, queue_id, op_attr, flow,
+			user_data, error);
+}
 int flow_get_flm_stats(struct flow_nic_dev *ndev, uint64_t *data, uint64_t size)
 {
 	const struct profile_inline_ops *profile_inline_ops = get_profile_inline_ops();
@@ -1113,6 +1150,8 @@ static const struct flow_filter_ops ops = {
 	 */
 	.flow_info_get = flow_info_get,
 	.flow_configure = flow_configure,
+	.flow_async_create = flow_async_create,
+	.flow_async_destroy = flow_async_destroy,
 
 	/*
 	 * Other
diff --git a/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_hw_db_inline.c b/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_hw_db_inline.c
index 2fee6ae6b5..ffab643f56 100644
--- a/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_hw_db_inline.c
+++ b/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_hw_db_inline.c
@@ -393,6 +393,19 @@ void hw_db_inline_deref_idxs(struct flow_nic_dev *ndev, void *db_handle, struct
 	}
 }
 
+struct hw_db_idx *hw_db_inline_find_idx(struct flow_nic_dev *ndev, void *db_handle,
+	enum hw_db_idx_type type, struct hw_db_idx *idxs, uint32_t size)
+{
+	(void)ndev;
+	(void)db_handle;
+	for (uint32_t i = 0; i < size; ++i) {
+		if (idxs[i].type == type)
+			return &idxs[i];
+	}
+
+	return NULL;
+}
+
 void hw_db_inline_dump(struct flow_nic_dev *ndev, void *db_handle, const struct hw_db_idx *idxs,
 	uint32_t size, FILE *file)
 {
diff --git a/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_hw_db_inline.h b/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_hw_db_inline.h
index c920d36cfd..aa046b68a7 100644
--- a/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_hw_db_inline.h
+++ b/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_hw_db_inline.h
@@ -287,6 +287,8 @@ void hw_db_inline_deref_idxs(struct flow_nic_dev *ndev, void *db_handle, struct
 	uint32_t size);
 const void *hw_db_inline_find_data(struct flow_nic_dev *ndev, void *db_handle,
 	enum hw_db_idx_type type, struct hw_db_idx *idxs, uint32_t size);
+struct hw_db_idx *hw_db_inline_find_idx(struct flow_nic_dev *ndev, void *db_handle,
+	enum hw_db_idx_type type, struct hw_db_idx *idxs, uint32_t size);
 void hw_db_inline_dump(struct flow_nic_dev *ndev, void *db_handle, const struct hw_db_idx *idxs,
 	uint32_t size, FILE *file);
 void hw_db_inline_dump_cfn(struct flow_nic_dev *ndev, void *db_handle, FILE *file);
diff --git a/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_profile_inline.c b/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_profile_inline.c
index f9133ad802..5d1244bddf 100644
--- a/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_profile_inline.c
+++ b/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_profile_inline.c
@@ -3,7 +3,6 @@
  * Copyright(c) 2023 Napatech A/S
  */
 
-#include "generic/rte_spinlock.h"
 #include "ntlog.h"
 #include "nt_util.h"
 
@@ -64,6 +63,11 @@
 #define POLICING_PARAMETER_OFFSET 4096
 #define SIZE_CONVERTER 1099.511627776
 
+#define CELL_STATUS_UNINITIALIZED 0
+#define CELL_STATUS_INITIALIZING 1
+#define CELL_STATUS_INITIALIZED_TYPE_FLOW 2
+#define CELL_STATUS_INITIALIZED_TYPE_FLM 3
+
 struct flm_mtr_stat_s {
 	struct dual_buckets_s *buckets;
 	atomic_uint_fast64_t n_pkt;
@@ -1034,6 +1038,17 @@ static int flm_flow_programming(struct flow_handle *fh, uint32_t flm_op)
 	return 0;
 }
 
+static inline const void *memcpy_or(void *dest, const void *src, size_t count)
+{
+	unsigned char *dest_ptr = (unsigned char *)dest;
+	const unsigned char *src_ptr = (const unsigned char *)src;
+
+	for (size_t i = 0; i < count; ++i)
+		dest_ptr[i] |= src_ptr[i];
+
+	return dest;
+}
+
 /*
  * This function must be callable without locking any mutexes
  */
@@ -4341,6 +4356,9 @@ int flow_destroy_profile_inline(struct flow_eth_dev *dev, struct flow_handle *fl
 {
 	int err = 0;
 
+	if (flow && flow->type == FLOW_HANDLE_TYPE_FLM && flow->flm_async)
+		return flow_async_destroy_profile_inline(dev, 0, NULL, flow, NULL, error);
+
 	flow_nic_set_error(ERR_SUCCESS, error);
 
 	if (flow) {
@@ -5485,6 +5503,232 @@ int flow_configure_profile_inline(struct flow_eth_dev *dev, uint8_t caller_id,
 	return -1;
 }
 
+struct flow_handle *flow_async_create_profile_inline(struct flow_eth_dev *dev,
+	uint32_t queue_id,
+	const struct rte_flow_op_attr *op_attr,
+	struct flow_template_table *template_table,
+	const struct rte_flow_item pattern[],
+	uint8_t pattern_template_index,
+	const struct rte_flow_action actions[],
+	uint8_t actions_template_index,
+	void *user_data,
+	struct rte_flow_error *error)
+{
+	(void)queue_id;
+	(void)op_attr;
+	struct flow_handle *fh = NULL;
+	int res, status;
+
+	const uint32_t pattern_action_index =
+		(uint32_t)template_table->nb_actions_templates * pattern_template_index +
+		actions_template_index;
+	struct flow_template_table_cell *pattern_action_pair =
+			&template_table->pattern_action_pairs[pattern_action_index];
+
+	uint32_t num_dest_port =
+		template_table->actions_templates[actions_template_index]->num_dest_port;
+	uint32_t num_queues =
+		template_table->actions_templates[actions_template_index]->num_queues;
+
+	uint32_t port_id = UINT32_MAX;
+	uint32_t packet_data[10];
+	uint32_t packet_mask[10];
+	struct flm_flow_key_def_s key_def;
+
+	flow_nic_set_error(ERR_SUCCESS, error);
+
+	struct nic_flow_def *fd = malloc(sizeof(struct nic_flow_def));
+
+	if (fd == NULL) {
+		error->type = RTE_FLOW_ERROR_TYPE_UNSPECIFIED;
+		error->message = "Failed to allocate flow_def";
+		goto err_exit;
+	}
+
+	memcpy(fd, template_table->actions_templates[actions_template_index]->fd,
+		sizeof(struct nic_flow_def));
+
+	res = interpret_flow_elements(dev, pattern, fd, error,
+			template_table->forced_vlan_vid, &port_id, packet_data,
+			packet_mask, &key_def);
+
+	if (res)
+		goto err_exit;
+
+	if (port_id == UINT32_MAX)
+		port_id = dev->port_id;
+
+	{
+		uint32_t num_dest_port_tmp = 0;
+		uint32_t num_queues_tmp = 0;
+
+		struct nic_flow_def action_fd = { 0 };
+		prepare_nic_flow_def(&action_fd);
+
+		res = interpret_flow_actions(dev, actions, NULL, &action_fd, error,
+				&num_dest_port_tmp, &num_queues_tmp);
+
+		if (res)
+			goto err_exit;
+
+		/* Copy FLM unique actions: modify_field, meter, encap/decap and age */
+		memcpy_or(fd->mtr_ids, action_fd.mtr_ids, sizeof(action_fd.mtr_ids));
+		memcpy_or(&fd->tun_hdr, &action_fd.tun_hdr, sizeof(struct tunnel_header_s));
+		memcpy_or(fd->modify_field, action_fd.modify_field,
+			sizeof(action_fd.modify_field));
+		fd->modify_field_count = action_fd.modify_field_count;
+		memcpy_or(&fd->age, &action_fd.age, sizeof(struct rte_flow_action_age));
+	}
+
+	status = atomic_load(&pattern_action_pair->status);
+
+	/* Initializing template entry */
+	if (status < CELL_STATUS_INITIALIZED_TYPE_FLOW) {
+		if (status == CELL_STATUS_UNINITIALIZED &&
+			atomic_compare_exchange_strong(&pattern_action_pair->status, &status,
+				CELL_STATUS_INITIALIZING)) {
+			rte_spinlock_lock(&dev->ndev->mtx);
+
+			fh = create_flow_filter(dev, fd, &template_table->attr,
+				template_table->forced_vlan_vid, template_table->caller_id,
+					error, port_id, num_dest_port, num_queues, packet_data,
+					packet_mask, &key_def);
+
+			rte_spinlock_unlock(&dev->ndev->mtx);
+
+			if (fh == NULL) {
+				/* reset status to CELL_STATUS_UNINITIALIZED to avoid a deadlock */
+				atomic_store(&pattern_action_pair->status,
+					CELL_STATUS_UNINITIALIZED);
+				goto err_exit;
+			}
+
+			if (fh->type == FLOW_HANDLE_TYPE_FLM) {
+				rte_spinlock_lock(&dev->ndev->mtx);
+
+				struct hw_db_idx *flm_ft_idx =
+					hw_db_inline_find_idx(dev->ndev, dev->ndev->hw_db_handle,
+						HW_DB_IDX_TYPE_FLM_FT,
+						(struct hw_db_idx *)fh->flm_db_idxs,
+						fh->flm_db_idx_counter);
+
+				rte_spinlock_unlock(&dev->ndev->mtx);
+
+				pattern_action_pair->flm_db_idx_counter = fh->flm_db_idx_counter;
+				memcpy(pattern_action_pair->flm_db_idxs, fh->flm_db_idxs,
+					sizeof(struct hw_db_idx) * fh->flm_db_idx_counter);
+
+				pattern_action_pair->flm_key_id = fh->flm_kid;
+				pattern_action_pair->flm_ft = flm_ft_idx->id1;
+
+				pattern_action_pair->flm_rpl_ext_ptr = fh->flm_rpl_ext_ptr;
+				pattern_action_pair->flm_scrub_prof = fh->flm_scrub_prof;
+
+				atomic_store(&pattern_action_pair->status,
+					CELL_STATUS_INITIALIZED_TYPE_FLM);
+
+				/* increment template table cell reference */
+				atomic_fetch_add(&pattern_action_pair->counter, 1);
+				fh->template_table_cell = pattern_action_pair;
+				fh->flm_async = true;
+
+			} else {
+				atomic_store(&pattern_action_pair->status,
+					CELL_STATUS_INITIALIZED_TYPE_FLOW);
+			}
+
+		} else {
+			do {
+				nt_os_wait_usec(1);
+				status = atomic_load(&pattern_action_pair->status);
+			} while (status == CELL_STATUS_INITIALIZING);
+
+			/* error handling in case that create_flow_filter() will fail in the other
+			 * thread
+			 */
+			if (status == CELL_STATUS_UNINITIALIZED)
+				goto err_exit;
+		}
+	}
+
+	/* FLM learn */
+	if (fh == NULL && status == CELL_STATUS_INITIALIZED_TYPE_FLM) {
+		fh = calloc(1, sizeof(struct flow_handle));
+
+		fh->type = FLOW_HANDLE_TYPE_FLM;
+		fh->dev = dev;
+		fh->caller_id = template_table->caller_id;
+		fh->user_data = user_data;
+
+		copy_fd_to_fh_flm(fh, fd, packet_data, pattern_action_pair->flm_key_id,
+			pattern_action_pair->flm_ft,
+			pattern_action_pair->flm_rpl_ext_ptr,
+			pattern_action_pair->flm_scrub_prof,
+			template_table->attr.priority & 0x3);
+
+		free(fd);
+
+		flm_flow_programming(fh, NT_FLM_OP_LEARN);
+
+		nic_insert_flow_flm(dev->ndev, fh);
+
+		/* increment template table cell reference */
+		atomic_fetch_add(&pattern_action_pair->counter, 1);
+		fh->template_table_cell = pattern_action_pair;
+		fh->flm_async = true;
+
+	} else if (fh == NULL) {
+		rte_spinlock_lock(&dev->ndev->mtx);
+
+		fh = create_flow_filter(dev, fd, &template_table->attr,
+			template_table->forced_vlan_vid, template_table->caller_id,
+				error, port_id, num_dest_port, num_queues, packet_data,
+				packet_mask, &key_def);
+
+		rte_spinlock_unlock(&dev->ndev->mtx);
+
+		if (fh == NULL)
+			goto err_exit;
+	}
+
+	if (fh) {
+		fh->caller_id = template_table->caller_id;
+		fh->user_data = user_data;
+	}
+
+	return fh;
+
+err_exit:
+	free(fd);
+	free(fh);
+
+	return NULL;
+}
+
+int flow_async_destroy_profile_inline(struct flow_eth_dev *dev, uint32_t queue_id,
+	const struct rte_flow_op_attr *op_attr, struct flow_handle *flow,
+	void *user_data, struct rte_flow_error *error)
+{
+	(void)queue_id;
+	(void)op_attr;
+	(void)user_data;
+
+	if (flow->type == FLOW_HANDLE_TYPE_FLOW)
+		return flow_destroy_profile_inline(dev, flow, error);
+
+	if (flm_flow_programming(flow, NT_FLM_OP_UNLEARN)) {
+		NT_LOG(ERR, FILTER, "FAILED to destroy flow: %p", flow);
+		flow_nic_set_error(ERR_REMOVE_FLOW_FAILED, error);
+		return -1;
+	}
+
+	nic_remove_flow_flm(dev->ndev, flow);
+
+	free(flow);
+
+	return 0;
+}
+
 static const struct profile_inline_ops ops = {
 	/*
 	 * Management
@@ -5509,6 +5753,8 @@ static const struct profile_inline_ops ops = {
 	.flow_get_flm_stats_profile_inline = flow_get_flm_stats_profile_inline,
 	.flow_info_get_profile_inline = flow_info_get_profile_inline,
 	.flow_configure_profile_inline = flow_configure_profile_inline,
+	.flow_async_create_profile_inline = flow_async_create_profile_inline,
+	.flow_async_destroy_profile_inline = flow_async_destroy_profile_inline,
 	/*
 	 * NT Flow FLM Meter API
 	 */
diff --git a/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_profile_inline.h b/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_profile_inline.h
index 8a03be1ab7..b548142342 100644
--- a/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_profile_inline.h
+++ b/drivers/net/ntnic/nthw/flow_api/profile_inline/flow_api_profile_inline.h
@@ -69,6 +69,20 @@ int flow_nic_set_hasher_fields_inline(struct flow_nic_dev *ndev,
 
 int flow_get_flm_stats_profile_inline(struct flow_nic_dev *ndev, uint64_t *data, uint64_t size);
 
+/*
+ * RTE flow asynchronous operations functions
+ */
+
+struct flow_handle *flow_async_create_profile_inline(struct flow_eth_dev *dev, uint32_t queue_id,
+	const struct rte_flow_op_attr *op_attr,
+	struct flow_template_table *template_table, const struct rte_flow_item pattern[],
+	uint8_t pattern_template_index, const struct rte_flow_action actions[],
+	uint8_t actions_template_index, void *user_data, struct rte_flow_error *error);
+
+int flow_async_destroy_profile_inline(struct flow_eth_dev *dev, uint32_t queue_id,
+	const struct rte_flow_op_attr *op_attr, struct flow_handle *flow,
+	void *user_data, struct rte_flow_error *error);
+
 int flow_info_get_profile_inline(struct flow_eth_dev *dev, uint8_t caller_id,
 	struct rte_flow_port_info *port_info,
 	struct rte_flow_queue_info *queue_info, struct rte_flow_error *error);
diff --git a/drivers/net/ntnic/ntnic_mod_reg.h b/drivers/net/ntnic/ntnic_mod_reg.h
index 92856b81d5..e8e7090661 100644
--- a/drivers/net/ntnic/ntnic_mod_reg.h
+++ b/drivers/net/ntnic/ntnic_mod_reg.h
@@ -310,6 +310,21 @@ struct profile_inline_ops {
 		uint32_t nb_contexts,
 		struct rte_flow_error *error);
 
+	/*
+	 * RTE flow asynchronous operations functions
+	 */
+
+	struct flow_handle *(*flow_async_create_profile_inline)(struct flow_eth_dev *dev,
+		uint32_t queue_id, const struct rte_flow_op_attr *op_attr,
+		struct flow_template_table *template_table, const struct rte_flow_item pattern[],
+		uint8_t rte_pattern_template_index, const struct rte_flow_action actions[],
+		uint8_t rte_actions_template_index, void *user_data, struct rte_flow_error *error);
+
+	int (*flow_async_destroy_profile_inline)(struct flow_eth_dev *dev, uint32_t queue_id,
+		const struct rte_flow_op_attr *op_attr,
+		struct flow_handle *flow, void *user_data,
+		struct rte_flow_error *error);
+
 	int (*flow_nic_set_hasher_fields_inline)(struct flow_nic_dev *ndev,
 		int hsh_idx,
 		struct nt_eth_rss_conf rss_conf);
-- 
2.45.0