[PATCH 1/3] net/mlx5/hws: split the root rule creation and destruction

[PATCH 1/3] net/mlx5/hws: split the root rule creation and destruction

[Maayan Kashani]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="y", Size: 41847 bytes --]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

Split the root rule creation/destruction into two stages:
 - do the job (create/destroy rule)
 - generate completion

Completion generation is required for the usual HWS API.
The create/destroy functions that don't generate completion
are exposed in header file and will be used by the coming
backward-compatible API.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_rule.c | 49 +++++++++++++++++++++---------
 drivers/net/mlx5/hws/mlx5dr_rule.h |  7 +++++
 2 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c b/drivers/net/mlx5/hws/mlx5dr_rule.c
index 171a0bff38..550f00a4c1 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.c
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -680,15 +680,12 @@ static int mlx5dr_rule_destroy_hws(struct mlx5dr_rule *rule,
 	return 0;
 }
 
-static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
-				   struct mlx5dr_rule_attr *rule_attr,
-				   const struct rte_flow_item items[],
-				   uint8_t at_idx,
-				   struct mlx5dr_rule_action rule_actions[])
+int mlx5dr_rule_create_root_no_comp(struct mlx5dr_rule *rule,
+				    const struct rte_flow_item items[],
+				    uint8_t num_actions,
+				    struct mlx5dr_rule_action rule_actions[])
 {
 	struct mlx5dv_flow_matcher *dv_matcher = rule->matcher->dv_matcher;
-	uint8_t num_actions = rule->matcher->at[at_idx].num_actions;
-	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
 	struct mlx5dv_flow_match_parameters *value;
 	struct mlx5_flow_attr flow_attr = {0};
 	struct mlx5dv_flow_action_attr *attr;
@@ -750,9 +747,6 @@ static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
 						    num_actions,
 						    attr);
 
-	mlx5dr_rule_gen_comp(&ctx->send_queue[rule_attr->queue_id], rule, !rule->flow,
-			     rule_attr->user_data, MLX5DR_RULE_STATUS_CREATED);
-
 	simple_free(value);
 	simple_free(attr);
 
@@ -766,14 +760,41 @@ static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
 	return rte_errno;
 }
 
+static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
+				   struct mlx5dr_rule_attr *rule_attr,
+				   const struct rte_flow_item items[],
+				   uint8_t num_actions,
+				   struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
+	int ret;
+
+	ret = mlx5dr_rule_create_root_no_comp(rule, items,
+					      num_actions, rule_actions);
+	if (ret)
+		return rte_errno;
+
+	mlx5dr_rule_gen_comp(&ctx->send_queue[rule_attr->queue_id], rule, !rule->flow,
+			     rule_attr->user_data, MLX5DR_RULE_STATUS_CREATED);
+
+	return 0;
+}
+
+int mlx5dr_rule_destroy_root_no_comp(struct mlx5dr_rule *rule)
+{
+	if (rule->flow)
+		return ibv_destroy_flow(rule->flow);
+
+	return 0;
+}
+
 static int mlx5dr_rule_destroy_root(struct mlx5dr_rule *rule,
 				    struct mlx5dr_rule_attr *attr)
 {
 	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
-	int err = 0;
+	int err;
 
-	if (rule->flow)
-		err = ibv_destroy_flow(rule->flow);
+	err = mlx5dr_rule_destroy_root_no_comp(rule);
 
 	mlx5dr_rule_gen_comp(&ctx->send_queue[attr->queue_id], rule, err,
 			     attr->user_data, MLX5DR_RULE_STATUS_DELETED);
@@ -970,7 +991,7 @@ int mlx5dr_rule_create(struct mlx5dr_matcher *matcher,
 		ret = mlx5dr_rule_create_root(rule_handle,
 					      attr,
 					      items,
-					      at_idx,
+					      matcher->at[at_idx].num_actions,
 					      rule_actions);
 	else
 		ret = mlx5dr_rule_create_hws(rule_handle,
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.h b/drivers/net/mlx5/hws/mlx5dr_rule.h
index dffaec1c0f..bc542eb543 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.h
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.h
@@ -74,4 +74,11 @@ int mlx5dr_rule_move_hws_add(struct mlx5dr_rule *rule,
 
 bool mlx5dr_rule_move_in_progress(struct mlx5dr_rule *rule);
 
+int mlx5dr_rule_create_root_no_comp(struct mlx5dr_rule *rule,
+				    const struct rte_flow_item items[],
+				    uint8_t num_actions,
+				    struct mlx5dr_rule_action rule_actions[]);
+
+int mlx5dr_rule_destroy_root_no_comp(struct mlx5dr_rule *rule);
+
 #endif /* MLX5DR_RULE_H_ */
-- 
2.25.1


From 7ca7c2a11276a155277ecb856115c7f30ab7c254 Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn@nvidia.com>
Date: Sun, 31 Dec 2023 01:20:36 +0200
Subject: [PATCH 2/3] net/mlx5/hws: add support for action type LAST

Add support for MLX5DR_ACTION_TYP_LAST action type.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr.h        | 12 ++++++++++++
 drivers/net/mlx5/hws/mlx5dr_action.c |  9 +++++++++
 2 files changed, 21 insertions(+)

diff --git a/drivers/net/mlx5/hws/mlx5dr.h b/drivers/net/mlx5/hws/mlx5dr.h
index 86cd4f1db5..5ceb1a7b4b 100644
--- a/drivers/net/mlx5/hws/mlx5dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -677,6 +677,18 @@ struct mlx5dr_action *
 mlx5dr_action_create_tag(struct mlx5dr_context *ctx,
 			 uint32_t flags);
 
+/* Create direct rule LAST action.
+ *
+ * @param[in] ctx
+ *	The context in which the new action will be created.
+ * @param[in] flags
+ *	Action creation flags. (enum mlx5dr_action_flags)
+ * @return pointer to mlx5dr_action on success NULL otherwise.
+ */
+struct mlx5dr_action *
+mlx5dr_action_create_last(struct mlx5dr_context *ctx,
+			  uint32_t flags);
+
 /* Create direct rule counter action.
  *
  * @param[in] ctx
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.c b/drivers/net/mlx5/hws/mlx5dr_action.c
index 562fb5cbb4..90f2f17bd0 100644
--- a/drivers/net/mlx5/hws/mlx5dr_action.c
+++ b/drivers/net/mlx5/hws/mlx5dr_action.c
@@ -1319,6 +1319,13 @@ mlx5dr_action_create_tag(struct mlx5dr_context *ctx,
 	return NULL;
 }
 
+struct mlx5dr_action *
+mlx5dr_action_create_last(struct mlx5dr_context *ctx,
+			  uint32_t flags)
+{
+	return mlx5dr_action_create_generic(ctx, flags, MLX5DR_ACTION_TYP_LAST);
+}
+
 static struct mlx5dr_action *
 mlx5dr_action_create_aso(struct mlx5dr_context *ctx,
 			 enum mlx5dr_action_type action_type,
@@ -3005,6 +3012,8 @@ static void mlx5dr_action_destroy_hws(struct mlx5dr_action *action)
 		for (i = 0; i < MLX5DR_ACTION_NAT64_STAGES; i++)
 			mlx5dr_action_destroy(action->nat64.stages[i]);
 		break;
+	case MLX5DR_ACTION_TYP_LAST:
+		break;
 	}
 }
 
-- 
2.25.1


From c20a2869be2409c807e68a25ecf35f3fb208c1d5 Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn@nvidia.com>
Date: Sun, 31 Dec 2023 01:24:07 +0200
Subject: [PATCH 3/3] net/mlx5/hws: add support for backward-compatible API
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

HWS API is very different from SWS and has limitations:
 - Queue based async insertion/deletion
 - Doesn’t handle complex rules
 - Requires the user to specify fixed matcher size – no rehash
 - Requires the user to specify action combination on matcher creation
 - Matching fields are passed using items and not FW/PRM format

These problems need to be solved in order to support HWS with
the old rte_flow API.

This patch adds backward-compatible (BWC) HWS API to allow for the
use of HWS in rte_flow and support the existing template-based API
alongside with the rte_flow API.

The new BWC API comprises mainly of the following four functions:
 - mlx5dr_bwc_matcher_create
 - mlx5dr_bwc_matcher_destroy
 - mlx5dr_bwc_rule_create
 - mlx5dr_bwc_rule_destroy

Notes:
 - To enable BWC API support, need to turn on 'bwc' flag in context
   attributes when calling mlx5dr_context_open()
 - BWC functions use their own queues
 - BWC functions provide synchronous API - they add/remove rule and
   poll for completion
 - BWC functions handle all the required matchers and rules allocations
 - For non-root tables, BWC functions use matcher resize and matcher
   attach action template features

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/meson.build       |   1 +
 drivers/net/mlx5/hws/mlx5dr.h          |  75 +++
 drivers/net/mlx5/hws/mlx5dr_bwc.c      | 898 +++++++++++++++++++++++++
 drivers/net/mlx5/hws/mlx5dr_bwc.h      |  34 +
 drivers/net/mlx5/hws/mlx5dr_context.c  |   3 +
 drivers/net/mlx5/hws/mlx5dr_context.h  |   7 +
 drivers/net/mlx5/hws/mlx5dr_internal.h |   1 +
 drivers/net/mlx5/hws/mlx5dr_send.c     |  43 +-
 8 files changed, 1061 insertions(+), 1 deletion(-)
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.c
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.h

diff --git a/drivers/net/mlx5/hws/meson.build b/drivers/net/mlx5/hws/meson.build
index bbcc628557..859a87fb85 100644
--- a/drivers/net/mlx5/hws/meson.build
+++ b/drivers/net/mlx5/hws/meson.build
@@ -20,4 +20,5 @@ sources += files(
         'mlx5dr_debug.c',
         'mlx5dr_pat_arg.c',
         'mlx5dr_crc32.c',
+        'mlx5dr_bwc.c',
 )
diff --git a/drivers/net/mlx5/hws/mlx5dr.h b/drivers/net/mlx5/hws/mlx5dr.h
index 5ceb1a7b4b..0fe39e9c76 100644
--- a/drivers/net/mlx5/hws/mlx5dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -106,6 +106,7 @@ struct mlx5dr_context_attr {
 	struct ibv_pd *pd;
 	/* Optional other ctx for resources allocation, all objects will be created on it */
 	struct ibv_context *shared_ibv_ctx;
+	bool bwc; /* add support for backward compatible API*/
 };
 
 struct mlx5dr_table_attr {
@@ -992,4 +993,78 @@ int mlx5dr_crc_encap_entropy_hash_calc(struct mlx5dr_context *ctx,
 				       uint8_t entropy_res[],
 				       enum mlx5dr_crc_encap_entropy_hash_size res_size);
 
+struct mlx5dr_bwc_matcher;
+struct mlx5dr_bwc_rule;
+
+/* Create a new BWC direct rule matcher.
+ * This function does the following:
+ *   - creates match template based on flow items
+ *   - creates an empty action template
+ *   - creates a usual mlx5dr_matcher with these mt and at, setting
+ *     its size to minimal
+ * Notes:
+ *   - table->ctx must have BWC support
+ *   - complex rules are not supported
+ *
+ * @param[in] table
+ *	The table in which the new matcher will be opened
+ * @param[in] priority
+ *	Priority for this BWC matcher
+ * @param[in] flow_items
+ *	Array of flow items that serve as basis for match and action templates
+ * @return pointer to mlx5dr_bwc_matcher on success or NULL otherwise.
+ */
+struct mlx5dr_bwc_matcher *
+mlx5dr_bwc_matcher_create(struct mlx5dr_table *table,
+			  uint32_t priority,
+			  const struct rte_flow_item flow_items[]);
+
+/* Destroy BWC direct rule matcher.
+ *
+ * @param[in] bwc_matcher
+ *	Matcher to destroy
+ * @return zero on success, non zero otherwise
+ */
+int mlx5dr_bwc_matcher_destroy(struct mlx5dr_bwc_matcher *bwc_matcher);
+
+/* Create a new BWC rule.
+ * Unlike the usual rule creation function, this one is blocking: when the
+ * function returns, the rule is written to its place (no need to poll).
+ * This function does the following:
+ *   - finds matching action template based on the provided rule_actions, or
+ *     creates new action template if matching action template doesn't exist
+ *   - updates corresponding BWC matcher stats
+ *   - if needed, the function performs rehash:
+ *       - creates a new matcher based on mt, at, new_sz
+ *       - moves all the existing matcher rules to the new matcher
+ *       - removes the old matcher
+ *   - inserts new rule
+ *   - polls till completion is received
+ * Notes:
+ *   - matcher->tbl->ctx must have BWC support
+ *   - separate BWC ctx queues are used
+ *
+ * @param[in] bwc_matcher
+ *	The BWC matcher in which the new rule will be created.
+ * @param[in] flow_items
+ *	Flow items to be used for the value matching
+ * @param[in] rule_actions
+ *	Rule action to be executed on match
+ * @param[in, out] rule_handle
+ *	A valid rule handle. The handle doesn't require any initialization
+ * @return valid BWC rule handle on success, NULL otherwise
+ */
+struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create(struct mlx5dr_bwc_matcher *bwc_matcher,
+		       const struct rte_flow_item flow_items[],
+		       struct mlx5dr_rule_action rule_actions[]);
+
+/* Destroy BWC direct rule.
+ *
+ * @param[in] bwc_rule
+ *	Rule to destroy
+ * @return zero on success, non zero otherwise
+ */
+int mlx5dr_bwc_rule_destroy(struct mlx5dr_bwc_rule *bwc_rule);
+
 #endif
diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.c b/drivers/net/mlx5/hws/mlx5dr_bwc.c
new file mode 100644
index 0000000000..eef3053ee0
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.c
@@ -0,0 +1,898 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static uint16_t mlx5dr_bwc_queues(struct mlx5dr_context *ctx)
+{
+	return (ctx->queues - 1) / 2;
+}
+
+static uint16_t
+mlx5dr_bwc_gen_queue_idx(struct mlx5dr_context *ctx)
+{
+	/* assign random queue */
+	return rand() % mlx5dr_bwc_queues(ctx);
+}
+
+static uint16_t
+mlx5dr_bwc_get_queue_id(struct mlx5dr_context *ctx, uint16_t idx)
+{
+	return idx + mlx5dr_bwc_queues(ctx);
+}
+
+static rte_spinlock_t *
+mlx5dr_bwc_get_queue_lock(struct mlx5dr_context *ctx, uint16_t idx)
+{
+	return &ctx->bwc_send_queue_locks[idx];
+}
+
+static void mlx5dr_bwc_lock_all_queues(struct mlx5dr_context *ctx)
+{
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	rte_spinlock_t *queue_lock;
+	int i;
+
+	for (i = 0; i < bwc_queues; i++) {
+		queue_lock = mlx5dr_bwc_get_queue_lock(ctx, i);
+		rte_spinlock_lock(queue_lock);
+	}
+}
+
+static void mlx5dr_bwc_unlock_all_queues(struct mlx5dr_context *ctx)
+{
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	rte_spinlock_t *queue_lock;
+	int i;
+
+	for (i = 0; i < bwc_queues; i++) {
+		queue_lock = mlx5dr_bwc_get_queue_lock(ctx, i);
+		rte_spinlock_unlock(queue_lock);
+	}
+}
+
+static void mlx5dr_bwc_matcher_init_attr(struct mlx5dr_matcher_attr *attr,
+					 uint32_t priority,
+					 uint8_t size_log,
+					 bool is_root)
+{
+	memset(attr, 0, sizeof(*attr));
+
+	attr->priority = priority;
+	attr->optimize_using_rule_idx = 0;
+	attr->mode = MLX5DR_MATCHER_RESOURCE_MODE_RULE;
+	attr->optimize_flow_src = MLX5DR_MATCHER_FLOW_SRC_ANY;
+	attr->insert_mode = MLX5DR_MATCHER_INSERT_BY_HASH;
+	attr->distribute_mode = MLX5DR_MATCHER_DISTRIBUTE_BY_HASH;
+	attr->rule.num_log = size_log;
+
+	if (!is_root) {
+		attr->resizable = true;
+		attr->max_num_of_at_attach = MLX5DR_BWC_MATCHER_ATTACH_AT_NUM;
+	}
+}
+
+struct mlx5dr_bwc_matcher *
+mlx5dr_bwc_matcher_create(struct mlx5dr_table *table,
+			  uint32_t priority,
+			  const struct rte_flow_item flow_items[])
+{
+	enum mlx5dr_action_type init_action_types[1] = { MLX5DR_ACTION_TYP_LAST };
+	uint16_t bwc_queues = mlx5dr_bwc_queues(table->ctx);
+	struct mlx5dr_bwc_matcher *bwc_matcher;
+	struct mlx5dr_matcher_attr attr = {0};
+	int i;
+
+	if (!mlx5dr_context_bwc_supported(table->ctx)) {
+		rte_errno = EINVAL;
+		DR_LOG(ERR, "BWC rule: Context created w/o BWC API compatibility");
+		return NULL;
+	}
+
+	bwc_matcher = simple_calloc(1, sizeof(*bwc_matcher));
+	if (!bwc_matcher) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	bwc_matcher->rules = simple_calloc(bwc_queues, sizeof(*bwc_matcher->rules));
+	if (!bwc_matcher->rules) {
+		rte_errno = ENOMEM;
+		goto free_bwc_matcher;
+	}
+
+	for (i = 0; i < bwc_queues; i++)
+		LIST_INIT(&bwc_matcher->rules[i]);
+
+	mlx5dr_bwc_matcher_init_attr(&attr,
+				     priority,
+				     MLX5DR_BWC_MATCHER_INIT_SIZE_LOG,
+				     mlx5dr_table_is_root(table));
+
+	bwc_matcher->mt = mlx5dr_match_template_create(flow_items,
+						       MLX5DR_MATCH_TEMPLATE_FLAG_NONE);
+	if (!bwc_matcher->mt) {
+		rte_errno = EINVAL;
+		goto free_bwc_matcher_rules;
+	}
+
+	bwc_matcher->priority = priority;
+	bwc_matcher->size_log = MLX5DR_BWC_MATCHER_INIT_SIZE_LOG;
+
+	/* create dummy action template */
+	bwc_matcher->at[0] = mlx5dr_action_template_create(init_action_types, 0);
+	bwc_matcher->num_of_at = 1;
+
+	bwc_matcher->matcher = mlx5dr_matcher_create(table,
+						     &bwc_matcher->mt, 1,
+						     &bwc_matcher->at[0],
+						     bwc_matcher->num_of_at,
+						     &attr);
+	if (!bwc_matcher->matcher) {
+		rte_errno = EINVAL;
+		goto free_at;
+	}
+
+	return bwc_matcher;
+
+free_at:
+	mlx5dr_action_template_destroy(bwc_matcher->at[0]);
+	mlx5dr_match_template_destroy(bwc_matcher->mt);
+free_bwc_matcher_rules:
+	simple_free(bwc_matcher->rules);
+free_bwc_matcher:
+	simple_free(bwc_matcher);
+
+	return NULL;
+}
+
+int mlx5dr_bwc_matcher_destroy(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	int i;
+
+	if (bwc_matcher->num_of_rules)
+		DR_LOG(ERR, "BWC matcher destroy: matcher still has %d rules",
+		       bwc_matcher->num_of_rules);
+
+	mlx5dr_matcher_destroy(bwc_matcher->matcher);
+	bwc_matcher->matcher = NULL;
+
+	for (i = 0; i < bwc_matcher->num_of_at; i++)
+		mlx5dr_action_template_destroy(bwc_matcher->at[i]);
+
+	mlx5dr_match_template_destroy(bwc_matcher->mt);
+	simple_free(bwc_matcher->rules);
+	simple_free(bwc_matcher);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_queue_poll(struct mlx5dr_context *ctx,
+		      uint16_t queue_id,
+		      uint32_t *pending_rules,
+		      bool drain)
+{
+	bool queue_full = *pending_rules == MLX5DR_BWC_MATCHER_REHASH_QUEUE_SZ;
+	bool got_comp = *pending_rules >= MLX5DR_BWC_MATCHER_REHASH_BURST_TH;
+	struct rte_flow_op_result comp[MLX5DR_BWC_MATCHER_REHASH_BURST_TH];
+	int ret;
+	int i;
+
+	/* Check if there are any completions at all */
+	if (!got_comp && !drain)
+		return 0;
+
+	while (queue_full || ((got_comp || drain) && *pending_rules)) {
+		ret = mlx5dr_send_queue_poll(ctx, queue_id, comp,
+					     MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+		if (unlikely(ret < 0)) {
+			DR_LOG(ERR, "Rehash error: polling queue %d returned %d\n",
+			       queue_id, ret);
+			return -EINVAL;
+		}
+
+		if (ret) {
+			(*pending_rules) -= ret;
+			for (i = 0; i < ret; i++) {
+				if (unlikely(comp[i].status != RTE_FLOW_OP_SUCCESS))
+					DR_LOG(ERR,
+					       "Rehash error: polling queue %d returned completion with error\n",
+					       queue_id);
+			}
+			queue_full = false;
+		}
+
+		got_comp = !!ret;
+	}
+
+	return 0;
+}
+
+static void
+mlx5dr_bwc_rule_fill_attr(struct mlx5dr_bwc_matcher *bwc_matcher,
+			  uint16_t bwc_queue_idx,
+			  struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+
+	/* no use of INSERT_BY_INDEX in bwc rule */
+	rule_attr->rule_idx = 0;
+
+	/* notify HW at each rule insertion/deletion */
+	rule_attr->burst = 0;
+
+	/* We don't need user data, but the API requires it to exist */
+	rule_attr->user_data = (void *)0xFACADE;
+
+	rule_attr->queue_id = mlx5dr_bwc_get_queue_id(ctx, bwc_queue_idx);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_alloc(void)
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+
+	bwc_rule = simple_calloc(1, sizeof(*bwc_rule));
+	if (unlikely(!bwc_rule))
+		goto out_err;
+
+	bwc_rule->rule = simple_calloc(1, sizeof(*bwc_rule->rule));
+	if (unlikely(!bwc_rule->rule))
+		goto free_rule;
+
+	return bwc_rule;
+
+free_rule:
+	simple_free(bwc_rule);
+out_err:
+	rte_errno = ENOMEM;
+	return NULL;
+}
+
+static void
+mlx5dr_bwc_rule_free(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	if (likely(bwc_rule->rule))
+		simple_free(bwc_rule->rule);
+	simple_free(bwc_rule);
+}
+
+static void
+mlx5dr_bwc_rule_list_add(struct mlx5dr_bwc_rule *bwc_rule, uint16_t idx)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+
+	rte_atomic_fetch_add_explicit(&bwc_matcher->num_of_rules, 1, rte_memory_order_relaxed);
+	bwc_rule->bwc_queue_idx = idx;
+	LIST_INSERT_HEAD(&bwc_matcher->rules[idx], bwc_rule, next);
+}
+
+static void mlx5dr_bwc_rule_list_remove(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+
+	rte_atomic_fetch_sub_explicit(&bwc_matcher->num_of_rules, 1, rte_memory_order_relaxed);
+	LIST_REMOVE(bwc_rule, next);
+}
+
+static int
+mlx5dr_bwc_rule_destroy_hws_async(struct mlx5dr_bwc_rule *bwc_rule,
+				  struct mlx5dr_rule_attr *attr)
+{
+	return mlx5dr_rule_destroy(bwc_rule->rule, attr);
+}
+
+static int
+mlx5dr_bwc_rule_destroy_hws_sync(struct mlx5dr_bwc_rule *bwc_rule,
+				 struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx;
+	struct rte_flow_op_result completion;
+	int ret;
+
+	ret = mlx5dr_bwc_rule_destroy_hws_async(bwc_rule, rule_attr);
+	if (unlikely(ret))
+		return ret;
+
+	do {
+		ret = mlx5dr_send_queue_poll(ctx, rule_attr->queue_id, &completion, 1);
+	} while (ret != 1);
+
+	if (unlikely(completion.status != RTE_FLOW_OP_SUCCESS ||
+		     (bwc_rule->rule->status != MLX5DR_RULE_STATUS_DELETED &&
+		      bwc_rule->rule->status != MLX5DR_RULE_STATUS_DELETING))) {
+		DR_LOG(ERR, "Failed destroying BWC rule: completion %d, rule status %d",
+		       completion.status, bwc_rule->rule->status);
+		rte_errno = EINVAL;
+		return rte_errno;
+	}
+
+	return 0;
+}
+
+static int mlx5dr_bwc_rule_destroy_hws(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	uint16_t idx = bwc_rule->bwc_queue_idx;
+	struct mlx5dr_rule_attr attr;
+	rte_spinlock_t *queue_lock;
+	int ret;
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, idx, &attr);
+
+	queue_lock = mlx5dr_bwc_get_queue_lock(ctx, idx);
+
+	rte_spinlock_lock(queue_lock);
+
+	ret = mlx5dr_bwc_rule_destroy_hws_sync(bwc_rule, &attr);
+	mlx5dr_bwc_rule_list_remove(bwc_rule);
+
+	rte_spinlock_unlock(queue_lock);
+
+	mlx5dr_bwc_rule_free(bwc_rule);
+
+	return ret;
+}
+
+static int mlx5dr_bwc_rule_destroy_root(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	int ret;
+
+	ret = mlx5dr_rule_destroy_root_no_comp(bwc_rule->rule);
+
+	mlx5dr_bwc_rule_free(bwc_rule);
+
+	return ret;
+}
+
+int mlx5dr_bwc_rule_destroy(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	if (unlikely(mlx5dr_table_is_root(bwc_rule->bwc_matcher->matcher->tbl)))
+		return mlx5dr_bwc_rule_destroy_root(bwc_rule);
+
+	return mlx5dr_bwc_rule_destroy_hws(bwc_rule);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws_async(struct mlx5dr_bwc_matcher *bwc_matcher,
+				 const struct rte_flow_item flow_items[],
+				 uint8_t at_idx,
+				 struct mlx5dr_rule_action rule_actions[],
+				 struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_alloc();
+	if (unlikely(!bwc_rule))
+		return NULL;
+
+	bwc_rule->bwc_matcher = bwc_matcher;
+
+	ret = mlx5dr_rule_create(bwc_matcher->matcher,
+				 0, /* only one match template supported */
+				 flow_items,
+				 at_idx,
+				 rule_actions,
+				 rule_attr,
+				 bwc_rule->rule);
+
+	if (unlikely(ret)) {
+		mlx5dr_bwc_rule_free(bwc_rule);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_root_sync(struct mlx5dr_bwc_matcher *bwc_matcher,
+				 const struct rte_flow_item flow_items[],
+				 uint8_t num_actions,
+				 struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_alloc();
+	if (unlikely(!bwc_rule)) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	bwc_rule->bwc_matcher = bwc_matcher;
+	bwc_rule->rule->matcher = bwc_matcher->matcher;
+
+	ret = mlx5dr_rule_create_root_no_comp(bwc_rule->rule,
+					      flow_items,
+					      num_actions,
+					      rule_actions);
+	if (unlikely(ret)) {
+		mlx5dr_bwc_rule_free(bwc_rule);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws_sync(struct mlx5dr_bwc_matcher *bwc_matcher,
+				const struct rte_flow_item flow_items[],
+				uint8_t at_idx,
+				struct mlx5dr_rule_action rule_actions[],
+				struct mlx5dr_rule_attr *rule_attr)
+
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	struct rte_flow_op_result completion;
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_async(bwc_matcher, flow_items,
+						    at_idx, rule_actions,
+						    rule_attr);
+	if (unlikely(!bwc_rule))
+		return NULL;
+
+	do {
+		ret = mlx5dr_send_queue_poll(ctx, rule_attr->queue_id, &completion, 1);
+	} while (ret != 1);
+
+	if (unlikely(completion.status != RTE_FLOW_OP_SUCCESS ||
+		     (bwc_rule->rule->status != MLX5DR_RULE_STATUS_CREATED &&
+		      bwc_rule->rule->status != MLX5DR_RULE_STATUS_CREATING))) {
+		DR_LOG(ERR, "Failed creating BWC rule: completion %d, rule status %d",
+		       completion.status, bwc_rule->rule->status);
+		mlx5dr_bwc_rule_free(bwc_rule);
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static bool
+mlx5dr_bwc_matcher_size_maxed_out(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps;
+
+	return bwc_matcher->size_log + MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH >=
+	       caps->ste_alloc_log_max - 1;
+}
+
+static bool
+mlx5dr_bwc_matcher_rehash_size_needed(struct mlx5dr_bwc_matcher *bwc_matcher,
+				      uint32_t num_of_rules)
+{
+	/* size-based rehash for root table is kernel's responsibility */
+	if (unlikely(mlx5dr_table_is_root(bwc_matcher->matcher->tbl)))
+		return false;
+
+	if (unlikely(mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher)))
+		return false;
+
+	if (unlikely((num_of_rules * 100 / MLX5DR_BWC_MATCHER_REHASH_PERCENT_TH) >=
+		     (1UL << bwc_matcher->size_log)))
+		return true;
+
+	return false;
+}
+
+static void
+mlx5dr_bwc_rule_actions_to_action_types(struct mlx5dr_rule_action rule_actions[],
+					enum mlx5dr_action_type action_types[])
+{
+	int i = 0;
+
+	for (i = 0;
+	     rule_actions[i].action && (rule_actions[i].action->type != MLX5DR_ACTION_TYP_LAST);
+	     i++) {
+		action_types[i] = (enum mlx5dr_action_type)rule_actions[i].action->type;
+	}
+
+	action_types[i] = MLX5DR_ACTION_TYP_LAST;
+}
+
+static int
+mlx5dr_bwc_rule_actions_num(struct mlx5dr_rule_action rule_actions[])
+{
+	int i = 0;
+
+	while (rule_actions[i].action &&
+	       (rule_actions[i].action->type != MLX5DR_ACTION_TYP_LAST))
+		i++;
+
+	return i;
+}
+
+static int
+mlx5dr_bwc_matcher_extend_at(struct mlx5dr_bwc_matcher *bwc_matcher,
+			     struct mlx5dr_rule_action rule_actions[])
+{
+	enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS];
+
+	mlx5dr_bwc_rule_actions_to_action_types(rule_actions, action_types);
+
+	bwc_matcher->at[bwc_matcher->num_of_at] =
+		mlx5dr_action_template_create(action_types, 0);
+
+	if (unlikely(!bwc_matcher->at[bwc_matcher->num_of_at])) {
+		rte_errno = ENOMEM;
+		return rte_errno;
+	}
+
+	bwc_matcher->num_of_at++;
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_extend_size(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps;
+
+	if (unlikely(mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher))) {
+		DR_LOG(ERR, "Can't resize matcher: depth exceeds limit %d",
+		       caps->rtc_log_depth_max);
+		return -ENOMEM;
+	}
+
+	bwc_matcher->size_log =
+		RTE_MIN(bwc_matcher->size_log + MLX5DR_BWC_MATCHER_SIZE_LOG_STEP,
+			caps->ste_alloc_log_max - MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_find_at(struct mlx5dr_bwc_matcher *bwc_matcher,
+			   struct mlx5dr_rule_action rule_actions[])
+{
+	enum mlx5dr_action_type *action_type_arr;
+	int i, j;
+
+	/* start from index 1 - first action template is a dummy */
+	for (i = 1; i < bwc_matcher->num_of_at; i++) {
+		j = 0;
+		action_type_arr = bwc_matcher->at[i]->action_type_arr;
+
+		while (rule_actions[j].action &&
+		       rule_actions[j].action->type != MLX5DR_ACTION_TYP_LAST) {
+			if (action_type_arr[j] != rule_actions[j].action->type)
+				break;
+			j++;
+		}
+
+		if (action_type_arr[j] == MLX5DR_ACTION_TYP_LAST &&
+		    (!rule_actions[j].action ||
+		     rule_actions[j].action->type == MLX5DR_ACTION_TYP_LAST))
+			return i;
+	}
+
+	return -1;
+}
+
+static int
+mlx5dr_bwc_matcher_move_all(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	struct mlx5dr_bwc_rule **bwc_rules;
+	struct mlx5dr_rule_attr rule_attr;
+	uint32_t *pending_rules;
+	bool all_done;
+	int i, j, ret;
+
+	if (mlx5dr_table_is_root(bwc_matcher->matcher->tbl)) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, 0, &rule_attr);
+
+	pending_rules = simple_calloc(bwc_queues, sizeof(*pending_rules));
+	if (!pending_rules) {
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+
+	bwc_rules = simple_calloc(bwc_queues, sizeof(*bwc_rules));
+	if (!bwc_rules) {
+		rte_errno = ENOMEM;
+		goto free_pending_rules;
+	}
+
+	for (i = 0; i < bwc_queues; i++) {
+		if (LIST_EMPTY(&bwc_matcher->rules[i]))
+			bwc_rules[i] = NULL;
+		else
+			bwc_rules[i] = LIST_FIRST(&bwc_matcher->rules[i]);
+	}
+
+	do {
+		all_done = true;
+
+		for (i = 0; i < bwc_queues; i++) {
+			rule_attr.queue_id = mlx5dr_bwc_get_queue_id(ctx, i);
+
+			for (j = 0; j < MLX5DR_BWC_MATCHER_REHASH_BURST_TH && bwc_rules[i]; j++) {
+				rule_attr.burst = !!((j + 1) % MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+				ret = mlx5dr_matcher_resize_rule_move(bwc_matcher->matcher,
+								      bwc_rules[i]->rule,
+								      &rule_attr);
+				if (ret) {
+					DR_LOG(ERR, "Moving BWC rule failed during rehash - %d",
+					       ret);
+					rte_errno = ENOMEM;
+					goto free_bwc_rules;
+				}
+
+				all_done = false;
+				pending_rules[i]++;
+				bwc_rules[i] = LIST_NEXT(bwc_rules[i], next);
+
+				mlx5dr_bwc_queue_poll(ctx, rule_attr.queue_id,
+						      &pending_rules[i], false);
+			}
+		}
+	} while (!all_done);
+
+	/* drain all the bwc queues */
+	for (i = 0; i < bwc_queues; i++) {
+		if (pending_rules[i]) {
+			uint16_t queue_id = mlx5dr_bwc_get_queue_id(ctx, i);
+			mlx5dr_send_engine_flush_queue(&ctx->send_queue[queue_id]);
+			mlx5dr_bwc_queue_poll(ctx, queue_id,
+					      &pending_rules[i], true);
+		}
+	}
+
+	rte_errno = 0;
+
+free_bwc_rules:
+	simple_free(bwc_rules);
+free_pending_rules:
+	simple_free(pending_rules);
+
+	return -rte_errno;
+}
+
+static int
+mlx5dr_bwc_matcher_move(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_matcher_attr matcher_attr = {0};
+	struct mlx5dr_matcher *old_matcher;
+	struct mlx5dr_matcher *new_matcher;
+	int ret;
+
+	mlx5dr_bwc_matcher_init_attr(&matcher_attr,
+				     bwc_matcher->priority,
+				     bwc_matcher->size_log,
+				     mlx5dr_table_is_root(bwc_matcher->matcher->tbl));
+
+	old_matcher = bwc_matcher->matcher;
+	new_matcher = mlx5dr_matcher_create(old_matcher->tbl,
+					    &bwc_matcher->mt, 1,
+					    bwc_matcher->at,
+					    bwc_matcher->num_of_at,
+					    &matcher_attr);
+	if (!new_matcher) {
+		DR_LOG(ERR, "Rehash error: matcher creation failed");
+		return -ENOMEM;
+	}
+
+	ret = mlx5dr_matcher_resize_set_target(old_matcher, new_matcher);
+	if (ret) {
+		DR_LOG(ERR, "Rehash error: failed setting resize target");
+		return ret;
+	}
+
+	ret = mlx5dr_bwc_matcher_move_all(bwc_matcher);
+	if (ret) {
+		DR_LOG(ERR, "Rehash error: moving rules failed");
+		return -ENOMEM;
+	}
+
+	bwc_matcher->matcher = new_matcher;
+	mlx5dr_matcher_destroy(old_matcher);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_rehash(struct mlx5dr_bwc_matcher *bwc_matcher, bool rehash_size)
+{
+	uint32_t num_of_rules;
+	int ret;
+
+	/* If the current matcher size is already at its max size, we can't
+	 * do the rehash. Skip it and try adding the rule again - perhaps
+	 * there was some change.
+	 */
+	if (mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher))
+		return 0;
+
+	/* It is possible that other rule has already performed rehash.
+	 * Need to check again if we really need rehash.
+	 * If the reason for rehash was size, but not any more - skip rehash.
+	 */
+	num_of_rules = rte_atomic_load_explicit(&bwc_matcher->num_of_rules,
+						rte_memory_order_relaxed);
+	if (rehash_size &&
+	    !mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules))
+		return 0;
+
+	/* Now we're done all the checking - do the rehash:
+	 *  - extend match RTC size
+	 *  - create new matcher
+	 *  - move all the rules to the new matcher
+	 *  - destroy the old matcher
+	 */
+
+	ret = mlx5dr_bwc_matcher_extend_size(bwc_matcher);
+	if (ret)
+		return ret;
+
+	return mlx5dr_bwc_matcher_move(bwc_matcher);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_root(struct mlx5dr_bwc_matcher *bwc_matcher,
+			    const struct rte_flow_item flow_items[],
+			    struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+
+	bwc_rule = mlx5dr_bwc_rule_create_root_sync(bwc_matcher,
+						    flow_items,
+						    mlx5dr_bwc_rule_actions_num(rule_actions),
+						    rule_actions);
+
+	if (unlikely(!bwc_rule))
+		DR_LOG(ERR, "BWC rule: failed creating rule on root tbl");
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws(struct mlx5dr_bwc_matcher *bwc_matcher,
+			   const struct rte_flow_item flow_items[],
+			   struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	struct mlx5dr_bwc_rule *bwc_rule = NULL;
+	struct mlx5dr_rule_attr rule_attr;
+	rte_spinlock_t *queue_lock;
+	bool rehash_size = false;
+	uint32_t num_of_rules;
+	uint16_t idx;
+	int at_idx;
+	int ret;
+
+	idx = mlx5dr_bwc_gen_queue_idx(ctx);
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, idx, &rule_attr);
+
+	queue_lock = mlx5dr_bwc_get_queue_lock(ctx, idx);
+
+	rte_spinlock_lock(queue_lock);
+
+	/* check if rehash needed due to missing action template */
+	at_idx = mlx5dr_bwc_matcher_find_at(bwc_matcher, rule_actions);
+	if (at_idx < 0) {
+		/* we need to extend BWC matcher action templates array */
+		rte_spinlock_unlock(queue_lock);
+		mlx5dr_bwc_lock_all_queues(ctx);
+
+		ret = mlx5dr_bwc_matcher_extend_at(bwc_matcher, rule_actions);
+		if (unlikely(ret)) {
+			mlx5dr_bwc_unlock_all_queues(ctx);
+			rte_errno = EINVAL;
+			DR_LOG(ERR, "BWC rule: failed extending action template - %d", ret);
+			return NULL;
+		}
+
+		/* action templates array was extended, we need the last idx */
+		at_idx = bwc_matcher->num_of_at - 1;
+
+		ret = mlx5dr_matcher_attach_at(bwc_matcher->matcher,
+					       bwc_matcher->at[at_idx]);
+		if (unlikely(ret)) {
+			mlx5dr_action_template_destroy(bwc_matcher->at[at_idx]);
+			bwc_matcher->at[at_idx] = NULL;
+			bwc_matcher->num_of_at--;
+
+			mlx5dr_bwc_unlock_all_queues(ctx);
+			DR_LOG(ERR, "BWC rule: failed attaching action template - %d", ret);
+			return NULL;
+		}
+
+		mlx5dr_bwc_unlock_all_queues(ctx);
+		rte_spinlock_lock(queue_lock);
+	}
+
+	/* check if number of rules require rehash */
+	num_of_rules = rte_atomic_load_explicit(&bwc_matcher->num_of_rules,
+						rte_memory_order_relaxed);
+	if (mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules)) {
+		rehash_size = true;
+		goto rehash;
+	}
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_sync(bwc_matcher,
+						   flow_items,
+						   at_idx,
+						   rule_actions,
+						   &rule_attr);
+
+	if (likely(bwc_rule)) {
+		mlx5dr_bwc_rule_list_add(bwc_rule, idx);
+		rte_spinlock_unlock(queue_lock);
+		return bwc_rule; /* rule inserted successfully */
+	}
+
+	/* At this point the rule wasn't added.
+	 * It could be because there was collision, or some other problem.
+	 * If we don't dive deeper than API, the only thing we know is that
+	 * the status of completion is RTE_FLOW_OP_ERROR.
+	 * Try rehash and insert rule again - last chance.
+	 */
+
+rehash:
+	rte_spinlock_unlock(queue_lock);
+
+	mlx5dr_bwc_lock_all_queues(ctx);
+	ret = mlx5dr_bwc_matcher_rehash(bwc_matcher, rehash_size);
+	mlx5dr_bwc_unlock_all_queues(ctx);
+
+	if (ret) {
+		DR_LOG(ERR, "BWC rule insertion: rehash failed - %d", ret);
+		return NULL;
+	}
+
+	/* Rehash done, but we still have that pesky rule to add */
+	rte_spinlock_lock(queue_lock);
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_sync(bwc_matcher,
+						   flow_items,
+						   at_idx,
+						   rule_actions,
+						   &rule_attr);
+
+	if (unlikely(!bwc_rule)) {
+		rte_spinlock_unlock(queue_lock);
+		DR_LOG(ERR, "BWC rule insertion failed");
+		return NULL;
+	}
+
+	mlx5dr_bwc_rule_list_add(bwc_rule, idx);
+	rte_spinlock_unlock(queue_lock);
+
+	return bwc_rule;
+}
+
+struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create(struct mlx5dr_bwc_matcher *bwc_matcher,
+		       const struct rte_flow_item flow_items[],
+		       struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+
+	if (unlikely(!mlx5dr_context_bwc_supported(ctx))) {
+		rte_errno = EINVAL;
+		DR_LOG(ERR, "BWC rule: Context created w/o BWC API compatibility");
+		return NULL;
+	}
+
+	if (unlikely(mlx5dr_table_is_root(bwc_matcher->matcher->tbl)))
+		return mlx5dr_bwc_rule_create_root(bwc_matcher,
+						   flow_items,
+						   rule_actions);
+
+	return mlx5dr_bwc_rule_create_hws(bwc_matcher,
+					  flow_items,
+					  rule_actions);
+}
diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.h b/drivers/net/mlx5/hws/mlx5dr_bwc.h
new file mode 100644
index 0000000000..648443861b
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5DR_BWC_H_
+#define MLX5DR_BWC_H_
+
+#define MLX5DR_BWC_MATCHER_INIT_SIZE_LOG 1
+#define MLX5DR_BWC_MATCHER_SIZE_LOG_STEP 1
+#define MLX5DR_BWC_MATCHER_REHASH_PERCENT_TH 70
+#define MLX5DR_BWC_MATCHER_REHASH_BURST_TH 32
+#define MLX5DR_BWC_MATCHER_REHASH_QUEUE_SZ 256
+#define MLX5DR_BWC_MATCHER_ATTACH_AT_NUM 255
+
+struct mlx5dr_bwc_matcher {
+	struct mlx5dr_matcher *matcher;
+	struct mlx5dr_match_template *mt;
+	struct mlx5dr_action_template *at[MLX5DR_BWC_MATCHER_ATTACH_AT_NUM];
+	uint8_t num_of_at;
+	uint32_t priority;
+	uint8_t size_log;
+	RTE_ATOMIC(uint32_t)num_of_rules; /* atomically accessed */
+	LIST_HEAD(rule_head, mlx5dr_bwc_rule) * rules;
+};
+
+struct mlx5dr_bwc_rule {
+	struct mlx5dr_bwc_matcher *bwc_matcher;
+	struct mlx5dr_rule *rule;
+	struct rte_flow_item *flow_items;
+	uint16_t bwc_queue_idx;
+	LIST_ENTRY(mlx5dr_bwc_rule) next;
+};
+
+#endif /* MLX5DR_BWC_H_ */
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.c b/drivers/net/mlx5/hws/mlx5dr_context.c
index 7f120b3b1b..db5e72927a 100644
--- a/drivers/net/mlx5/hws/mlx5dr_context.c
+++ b/drivers/net/mlx5/hws/mlx5dr_context.c
@@ -176,6 +176,9 @@ static int mlx5dr_context_init_hws(struct mlx5dr_context *ctx,
 	if (ret)
 		goto uninit_pd;
 
+	if (attr->bwc)
+		ctx->flags |= MLX5DR_CONTEXT_FLAG_BWC_SUPPORT;
+
 	ret = mlx5dr_send_queues_open(ctx, attr->queues, attr->queue_size);
 	if (ret)
 		goto pools_uninit;
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.h b/drivers/net/mlx5/hws/mlx5dr_context.h
index f476c2308c..7678232ecc 100644
--- a/drivers/net/mlx5/hws/mlx5dr_context.h
+++ b/drivers/net/mlx5/hws/mlx5dr_context.h
@@ -8,6 +8,7 @@
 enum mlx5dr_context_flags {
 	MLX5DR_CONTEXT_FLAG_HWS_SUPPORT = 1 << 0,
 	MLX5DR_CONTEXT_FLAG_PRIVATE_PD = 1 << 1,
+	MLX5DR_CONTEXT_FLAG_BWC_SUPPORT = 1 << 2,
 };
 
 enum mlx5dr_context_shared_stc_type {
@@ -44,6 +45,7 @@ struct mlx5dr_context {
 	enum mlx5dr_context_flags flags;
 	struct mlx5dr_send_engine *send_queue;
 	size_t queues;
+	rte_spinlock_t *bwc_send_queue_locks;
 	LIST_HEAD(table_head, mlx5dr_table) head;
 };
 
@@ -52,6 +54,11 @@ static inline bool mlx5dr_context_shared_gvmi_used(struct mlx5dr_context *ctx)
 	return ctx->local_ibv_ctx ? true : false;
 }
 
+static inline bool mlx5dr_context_bwc_supported(struct mlx5dr_context *ctx)
+{
+	return ctx->flags & MLX5DR_CONTEXT_FLAG_BWC_SUPPORT;
+}
+
 static inline struct ibv_context *
 mlx5dr_context_get_local_ibv(struct mlx5dr_context *ctx)
 {
diff --git a/drivers/net/mlx5/hws/mlx5dr_internal.h b/drivers/net/mlx5/hws/mlx5dr_internal.h
index b9efdc4a9a..2abc516b5e 100644
--- a/drivers/net/mlx5/hws/mlx5dr_internal.h
+++ b/drivers/net/mlx5/hws/mlx5dr_internal.h
@@ -39,6 +39,7 @@
 #include "mlx5dr_debug.h"
 #include "mlx5dr_pat_arg.h"
 #include "mlx5dr_crc32.h"
+#include "mlx5dr_bwc.h"
 
 #define W_SIZE		2
 #define DW_SIZE		4
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c b/drivers/net/mlx5/hws/mlx5dr_send.c
index 2942668e76..0120f03a48 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.c
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -971,10 +971,42 @@ static void __mlx5dr_send_queues_close(struct mlx5dr_context *ctx, uint16_t queu
 		mlx5dr_send_queue_close(&ctx->send_queue[queues]);
 }
 
+static int mlx5dr_bwc_send_queues_init(struct mlx5dr_context *ctx)
+{
+	int bwc_queues = ctx->queues - 1;
+	int i;
+
+	if (!mlx5dr_context_bwc_supported(ctx))
+		return 0;
+
+	ctx->queues += bwc_queues;
+
+	ctx->bwc_send_queue_locks = simple_calloc(bwc_queues,
+						  sizeof(*ctx->bwc_send_queue_locks));
+	if (!ctx->bwc_send_queue_locks) {
+		rte_errno = ENOMEM;
+		return rte_errno;
+	}
+
+	for (i = 0; i < bwc_queues; i++)
+		rte_spinlock_init(&ctx->bwc_send_queue_locks[i]);
+
+	return 0;
+}
+
+static void mlx5dr_send_queues_bwc_locks_destroy(struct mlx5dr_context *ctx)
+{
+	if (!mlx5dr_context_bwc_supported(ctx))
+		return;
+
+	simple_free(ctx->bwc_send_queue_locks);
+}
+
 void mlx5dr_send_queues_close(struct mlx5dr_context *ctx)
 {
 	__mlx5dr_send_queues_close(ctx, ctx->queues);
 	simple_free(ctx->send_queue);
+	mlx5dr_send_queues_bwc_locks_destroy(ctx);
 }
 
 int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
@@ -987,10 +1019,16 @@ int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
 	/* Open one extra queue for control path */
 	ctx->queues = queues + 1;
 
+	/* open a separate set of queues and locks for bwc API */
+	err = mlx5dr_bwc_send_queues_init(ctx);
+	if (err)
+		return err;
+
 	ctx->send_queue = simple_calloc(ctx->queues, sizeof(*ctx->send_queue));
 	if (!ctx->send_queue) {
 		rte_errno = ENOMEM;
-		return rte_errno;
+		err = rte_errno;
+		goto free_bwc_locks;
 	}
 
 	for (i = 0; i < ctx->queues; i++) {
@@ -1006,6 +1044,9 @@ int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
 
 	simple_free(ctx->send_queue);
 
+free_bwc_locks:
+	mlx5dr_send_queues_bwc_locks_destroy(ctx);
+
 	return err;
 }
 
-- 
2.25.1

[PATCH v3 1/3] net/mlx5/hws: split the root rule creation and destruction

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

Split the root rule creation/destruction into two stages:
 - do the job (create/destroy rule)
 - generate completion

Completion generation is required for the usual HWS API.
The create/destroy functions that don't generate completion
are exposed in header file and will be used by the coming
backward-compatible API.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_rule.c | 49 +++++++++++++++++++++---------
 drivers/net/mlx5/hws/mlx5dr_rule.h |  7 +++++
 2 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c b/drivers/net/mlx5/hws/mlx5dr_rule.c
index 171a0bff38..550f00a4c1 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.c
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -680,15 +680,12 @@ static int mlx5dr_rule_destroy_hws(struct mlx5dr_rule *rule,
 	return 0;
 }
 
-static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
-				   struct mlx5dr_rule_attr *rule_attr,
-				   const struct rte_flow_item items[],
-				   uint8_t at_idx,
-				   struct mlx5dr_rule_action rule_actions[])
+int mlx5dr_rule_create_root_no_comp(struct mlx5dr_rule *rule,
+				    const struct rte_flow_item items[],
+				    uint8_t num_actions,
+				    struct mlx5dr_rule_action rule_actions[])
 {
 	struct mlx5dv_flow_matcher *dv_matcher = rule->matcher->dv_matcher;
-	uint8_t num_actions = rule->matcher->at[at_idx].num_actions;
-	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
 	struct mlx5dv_flow_match_parameters *value;
 	struct mlx5_flow_attr flow_attr = {0};
 	struct mlx5dv_flow_action_attr *attr;
@@ -750,9 +747,6 @@ static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
 						    num_actions,
 						    attr);
 
-	mlx5dr_rule_gen_comp(&ctx->send_queue[rule_attr->queue_id], rule, !rule->flow,
-			     rule_attr->user_data, MLX5DR_RULE_STATUS_CREATED);
-
 	simple_free(value);
 	simple_free(attr);
 
@@ -766,14 +760,41 @@ static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
 	return rte_errno;
 }
 
+static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
+				   struct mlx5dr_rule_attr *rule_attr,
+				   const struct rte_flow_item items[],
+				   uint8_t num_actions,
+				   struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
+	int ret;
+
+	ret = mlx5dr_rule_create_root_no_comp(rule, items,
+					      num_actions, rule_actions);
+	if (ret)
+		return rte_errno;
+
+	mlx5dr_rule_gen_comp(&ctx->send_queue[rule_attr->queue_id], rule, !rule->flow,
+			     rule_attr->user_data, MLX5DR_RULE_STATUS_CREATED);
+
+	return 0;
+}
+
+int mlx5dr_rule_destroy_root_no_comp(struct mlx5dr_rule *rule)
+{
+	if (rule->flow)
+		return ibv_destroy_flow(rule->flow);
+
+	return 0;
+}
+
 static int mlx5dr_rule_destroy_root(struct mlx5dr_rule *rule,
 				    struct mlx5dr_rule_attr *attr)
 {
 	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
-	int err = 0;
+	int err;
 
-	if (rule->flow)
-		err = ibv_destroy_flow(rule->flow);
+	err = mlx5dr_rule_destroy_root_no_comp(rule);
 
 	mlx5dr_rule_gen_comp(&ctx->send_queue[attr->queue_id], rule, err,
 			     attr->user_data, MLX5DR_RULE_STATUS_DELETED);
@@ -970,7 +991,7 @@ int mlx5dr_rule_create(struct mlx5dr_matcher *matcher,
 		ret = mlx5dr_rule_create_root(rule_handle,
 					      attr,
 					      items,
-					      at_idx,
+					      matcher->at[at_idx].num_actions,
 					      rule_actions);
 	else
 		ret = mlx5dr_rule_create_hws(rule_handle,
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.h b/drivers/net/mlx5/hws/mlx5dr_rule.h
index dffaec1c0f..bc542eb543 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.h
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.h
@@ -74,4 +74,11 @@ int mlx5dr_rule_move_hws_add(struct mlx5dr_rule *rule,
 
 bool mlx5dr_rule_move_in_progress(struct mlx5dr_rule *rule);
 
+int mlx5dr_rule_create_root_no_comp(struct mlx5dr_rule *rule,
+				    const struct rte_flow_item items[],
+				    uint8_t num_actions,
+				    struct mlx5dr_rule_action rule_actions[]);
+
+int mlx5dr_rule_destroy_root_no_comp(struct mlx5dr_rule *rule);
+
 #endif /* MLX5DR_RULE_H_ */
-- 
2.25.1

[PATCH v3 2/3] net/mlx5/hws: add support for action type LAST

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

Add support for MLX5DR_ACTION_TYP_LAST action type.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr.h        | 12 ++++++++++++
 drivers/net/mlx5/hws/mlx5dr_action.c |  9 +++++++++
 2 files changed, 21 insertions(+)

diff --git a/drivers/net/mlx5/hws/mlx5dr.h b/drivers/net/mlx5/hws/mlx5dr.h
index 86cd4f1db5..5ceb1a7b4b 100644
--- a/drivers/net/mlx5/hws/mlx5dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -677,6 +677,18 @@ struct mlx5dr_action *
 mlx5dr_action_create_tag(struct mlx5dr_context *ctx,
 			 uint32_t flags);
 
+/* Create direct rule LAST action.
+ *
+ * @param[in] ctx
+ *	The context in which the new action will be created.
+ * @param[in] flags
+ *	Action creation flags. (enum mlx5dr_action_flags)
+ * @return pointer to mlx5dr_action on success NULL otherwise.
+ */
+struct mlx5dr_action *
+mlx5dr_action_create_last(struct mlx5dr_context *ctx,
+			  uint32_t flags);
+
 /* Create direct rule counter action.
  *
  * @param[in] ctx
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.c b/drivers/net/mlx5/hws/mlx5dr_action.c
index 562fb5cbb4..90f2f17bd0 100644
--- a/drivers/net/mlx5/hws/mlx5dr_action.c
+++ b/drivers/net/mlx5/hws/mlx5dr_action.c
@@ -1319,6 +1319,13 @@ mlx5dr_action_create_tag(struct mlx5dr_context *ctx,
 	return NULL;
 }
 
+struct mlx5dr_action *
+mlx5dr_action_create_last(struct mlx5dr_context *ctx,
+			  uint32_t flags)
+{
+	return mlx5dr_action_create_generic(ctx, flags, MLX5DR_ACTION_TYP_LAST);
+}
+
 static struct mlx5dr_action *
 mlx5dr_action_create_aso(struct mlx5dr_context *ctx,
 			 enum mlx5dr_action_type action_type,
@@ -3005,6 +3012,8 @@ static void mlx5dr_action_destroy_hws(struct mlx5dr_action *action)
 		for (i = 0; i < MLX5DR_ACTION_NAT64_STAGES; i++)
 			mlx5dr_action_destroy(action->nat64.stages[i]);
 		break;
+	case MLX5DR_ACTION_TYP_LAST:
+		break;
 	}
 }
 
-- 
2.25.1

[PATCH v3 3/3] net/mlx5/hws: add support for backward-compatible API

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

HWS API is very different from SWS and has limitations:
 - Queue based async insertion/deletion
 - Doesn’t handle complex rules
 - Requires the user to specify fixed matcher size – no rehash
 - Requires the user to specify action combination on matcher creation
 - Matching fields are passed using items and not FW/PRM format

These problems need to be solved in order to support HWS with
the old rte_flow API.

This patch adds backward-compatible (BWC) HWS API to allow for the
use of HWS in rte_flow and support the existing template-based API
alongside with the rte_flow API.

The new BWC API comprises mainly of the following four functions:
 - mlx5dr_bwc_matcher_create
 - mlx5dr_bwc_matcher_destroy
 - mlx5dr_bwc_rule_create
 - mlx5dr_bwc_rule_destroy

Notes:
 - To enable BWC API support, need to turn on 'bwc' flag in context
   attributes when calling mlx5dr_context_open()
 - BWC functions use their own queues
 - BWC functions provide synchronous API - they add/remove rule and
   poll for completion
 - BWC functions handle all the required matchers and rules allocations
 - For non-root tables, BWC functions use matcher resize and matcher
   attach action template features

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/meson.build       |   1 +
 drivers/net/mlx5/hws/mlx5dr.h          |  75 +++
 drivers/net/mlx5/hws/mlx5dr_bwc.c      | 898 +++++++++++++++++++++++++
 drivers/net/mlx5/hws/mlx5dr_bwc.h      |  34 +
 drivers/net/mlx5/hws/mlx5dr_context.c  |   3 +
 drivers/net/mlx5/hws/mlx5dr_context.h  |   7 +
 drivers/net/mlx5/hws/mlx5dr_internal.h |   1 +
 drivers/net/mlx5/hws/mlx5dr_send.c     |  43 +-
 8 files changed, 1061 insertions(+), 1 deletion(-)
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.c
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.h

diff --git a/drivers/net/mlx5/hws/meson.build b/drivers/net/mlx5/hws/meson.build
index bbcc628557..859a87fb85 100644
--- a/drivers/net/mlx5/hws/meson.build
+++ b/drivers/net/mlx5/hws/meson.build
@@ -20,4 +20,5 @@ sources += files(
         'mlx5dr_debug.c',
         'mlx5dr_pat_arg.c',
         'mlx5dr_crc32.c',
+        'mlx5dr_bwc.c',
 )
diff --git a/drivers/net/mlx5/hws/mlx5dr.h b/drivers/net/mlx5/hws/mlx5dr.h
index 5ceb1a7b4b..0fe39e9c76 100644
--- a/drivers/net/mlx5/hws/mlx5dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -106,6 +106,7 @@ struct mlx5dr_context_attr {
 	struct ibv_pd *pd;
 	/* Optional other ctx for resources allocation, all objects will be created on it */
 	struct ibv_context *shared_ibv_ctx;
+	bool bwc; /* add support for backward compatible API*/
 };
 
 struct mlx5dr_table_attr {
@@ -992,4 +993,78 @@ int mlx5dr_crc_encap_entropy_hash_calc(struct mlx5dr_context *ctx,
 				       uint8_t entropy_res[],
 				       enum mlx5dr_crc_encap_entropy_hash_size res_size);
 
+struct mlx5dr_bwc_matcher;
+struct mlx5dr_bwc_rule;
+
+/* Create a new BWC direct rule matcher.
+ * This function does the following:
+ *   - creates match template based on flow items
+ *   - creates an empty action template
+ *   - creates a usual mlx5dr_matcher with these mt and at, setting
+ *     its size to minimal
+ * Notes:
+ *   - table->ctx must have BWC support
+ *   - complex rules are not supported
+ *
+ * @param[in] table
+ *	The table in which the new matcher will be opened
+ * @param[in] priority
+ *	Priority for this BWC matcher
+ * @param[in] flow_items
+ *	Array of flow items that serve as basis for match and action templates
+ * @return pointer to mlx5dr_bwc_matcher on success or NULL otherwise.
+ */
+struct mlx5dr_bwc_matcher *
+mlx5dr_bwc_matcher_create(struct mlx5dr_table *table,
+			  uint32_t priority,
+			  const struct rte_flow_item flow_items[]);
+
+/* Destroy BWC direct rule matcher.
+ *
+ * @param[in] bwc_matcher
+ *	Matcher to destroy
+ * @return zero on success, non zero otherwise
+ */
+int mlx5dr_bwc_matcher_destroy(struct mlx5dr_bwc_matcher *bwc_matcher);
+
+/* Create a new BWC rule.
+ * Unlike the usual rule creation function, this one is blocking: when the
+ * function returns, the rule is written to its place (no need to poll).
+ * This function does the following:
+ *   - finds matching action template based on the provided rule_actions, or
+ *     creates new action template if matching action template doesn't exist
+ *   - updates corresponding BWC matcher stats
+ *   - if needed, the function performs rehash:
+ *       - creates a new matcher based on mt, at, new_sz
+ *       - moves all the existing matcher rules to the new matcher
+ *       - removes the old matcher
+ *   - inserts new rule
+ *   - polls till completion is received
+ * Notes:
+ *   - matcher->tbl->ctx must have BWC support
+ *   - separate BWC ctx queues are used
+ *
+ * @param[in] bwc_matcher
+ *	The BWC matcher in which the new rule will be created.
+ * @param[in] flow_items
+ *	Flow items to be used for the value matching
+ * @param[in] rule_actions
+ *	Rule action to be executed on match
+ * @param[in, out] rule_handle
+ *	A valid rule handle. The handle doesn't require any initialization
+ * @return valid BWC rule handle on success, NULL otherwise
+ */
+struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create(struct mlx5dr_bwc_matcher *bwc_matcher,
+		       const struct rte_flow_item flow_items[],
+		       struct mlx5dr_rule_action rule_actions[]);
+
+/* Destroy BWC direct rule.
+ *
+ * @param[in] bwc_rule
+ *	Rule to destroy
+ * @return zero on success, non zero otherwise
+ */
+int mlx5dr_bwc_rule_destroy(struct mlx5dr_bwc_rule *bwc_rule);
+
 #endif
diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.c b/drivers/net/mlx5/hws/mlx5dr_bwc.c
new file mode 100644
index 0000000000..eef3053ee0
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.c
@@ -0,0 +1,898 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static uint16_t mlx5dr_bwc_queues(struct mlx5dr_context *ctx)
+{
+	return (ctx->queues - 1) / 2;
+}
+
+static uint16_t
+mlx5dr_bwc_gen_queue_idx(struct mlx5dr_context *ctx)
+{
+	/* assign random queue */
+	return rand() % mlx5dr_bwc_queues(ctx);
+}
+
+static uint16_t
+mlx5dr_bwc_get_queue_id(struct mlx5dr_context *ctx, uint16_t idx)
+{
+	return idx + mlx5dr_bwc_queues(ctx);
+}
+
+static rte_spinlock_t *
+mlx5dr_bwc_get_queue_lock(struct mlx5dr_context *ctx, uint16_t idx)
+{
+	return &ctx->bwc_send_queue_locks[idx];
+}
+
+static void mlx5dr_bwc_lock_all_queues(struct mlx5dr_context *ctx)
+{
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	rte_spinlock_t *queue_lock;
+	int i;
+
+	for (i = 0; i < bwc_queues; i++) {
+		queue_lock = mlx5dr_bwc_get_queue_lock(ctx, i);
+		rte_spinlock_lock(queue_lock);
+	}
+}
+
+static void mlx5dr_bwc_unlock_all_queues(struct mlx5dr_context *ctx)
+{
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	rte_spinlock_t *queue_lock;
+	int i;
+
+	for (i = 0; i < bwc_queues; i++) {
+		queue_lock = mlx5dr_bwc_get_queue_lock(ctx, i);
+		rte_spinlock_unlock(queue_lock);
+	}
+}
+
+static void mlx5dr_bwc_matcher_init_attr(struct mlx5dr_matcher_attr *attr,
+					 uint32_t priority,
+					 uint8_t size_log,
+					 bool is_root)
+{
+	memset(attr, 0, sizeof(*attr));
+
+	attr->priority = priority;
+	attr->optimize_using_rule_idx = 0;
+	attr->mode = MLX5DR_MATCHER_RESOURCE_MODE_RULE;
+	attr->optimize_flow_src = MLX5DR_MATCHER_FLOW_SRC_ANY;
+	attr->insert_mode = MLX5DR_MATCHER_INSERT_BY_HASH;
+	attr->distribute_mode = MLX5DR_MATCHER_DISTRIBUTE_BY_HASH;
+	attr->rule.num_log = size_log;
+
+	if (!is_root) {
+		attr->resizable = true;
+		attr->max_num_of_at_attach = MLX5DR_BWC_MATCHER_ATTACH_AT_NUM;
+	}
+}
+
+struct mlx5dr_bwc_matcher *
+mlx5dr_bwc_matcher_create(struct mlx5dr_table *table,
+			  uint32_t priority,
+			  const struct rte_flow_item flow_items[])
+{
+	enum mlx5dr_action_type init_action_types[1] = { MLX5DR_ACTION_TYP_LAST };
+	uint16_t bwc_queues = mlx5dr_bwc_queues(table->ctx);
+	struct mlx5dr_bwc_matcher *bwc_matcher;
+	struct mlx5dr_matcher_attr attr = {0};
+	int i;
+
+	if (!mlx5dr_context_bwc_supported(table->ctx)) {
+		rte_errno = EINVAL;
+		DR_LOG(ERR, "BWC rule: Context created w/o BWC API compatibility");
+		return NULL;
+	}
+
+	bwc_matcher = simple_calloc(1, sizeof(*bwc_matcher));
+	if (!bwc_matcher) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	bwc_matcher->rules = simple_calloc(bwc_queues, sizeof(*bwc_matcher->rules));
+	if (!bwc_matcher->rules) {
+		rte_errno = ENOMEM;
+		goto free_bwc_matcher;
+	}
+
+	for (i = 0; i < bwc_queues; i++)
+		LIST_INIT(&bwc_matcher->rules[i]);
+
+	mlx5dr_bwc_matcher_init_attr(&attr,
+				     priority,
+				     MLX5DR_BWC_MATCHER_INIT_SIZE_LOG,
+				     mlx5dr_table_is_root(table));
+
+	bwc_matcher->mt = mlx5dr_match_template_create(flow_items,
+						       MLX5DR_MATCH_TEMPLATE_FLAG_NONE);
+	if (!bwc_matcher->mt) {
+		rte_errno = EINVAL;
+		goto free_bwc_matcher_rules;
+	}
+
+	bwc_matcher->priority = priority;
+	bwc_matcher->size_log = MLX5DR_BWC_MATCHER_INIT_SIZE_LOG;
+
+	/* create dummy action template */
+	bwc_matcher->at[0] = mlx5dr_action_template_create(init_action_types, 0);
+	bwc_matcher->num_of_at = 1;
+
+	bwc_matcher->matcher = mlx5dr_matcher_create(table,
+						     &bwc_matcher->mt, 1,
+						     &bwc_matcher->at[0],
+						     bwc_matcher->num_of_at,
+						     &attr);
+	if (!bwc_matcher->matcher) {
+		rte_errno = EINVAL;
+		goto free_at;
+	}
+
+	return bwc_matcher;
+
+free_at:
+	mlx5dr_action_template_destroy(bwc_matcher->at[0]);
+	mlx5dr_match_template_destroy(bwc_matcher->mt);
+free_bwc_matcher_rules:
+	simple_free(bwc_matcher->rules);
+free_bwc_matcher:
+	simple_free(bwc_matcher);
+
+	return NULL;
+}
+
+int mlx5dr_bwc_matcher_destroy(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	int i;
+
+	if (bwc_matcher->num_of_rules)
+		DR_LOG(ERR, "BWC matcher destroy: matcher still has %d rules",
+		       bwc_matcher->num_of_rules);
+
+	mlx5dr_matcher_destroy(bwc_matcher->matcher);
+	bwc_matcher->matcher = NULL;
+
+	for (i = 0; i < bwc_matcher->num_of_at; i++)
+		mlx5dr_action_template_destroy(bwc_matcher->at[i]);
+
+	mlx5dr_match_template_destroy(bwc_matcher->mt);
+	simple_free(bwc_matcher->rules);
+	simple_free(bwc_matcher);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_queue_poll(struct mlx5dr_context *ctx,
+		      uint16_t queue_id,
+		      uint32_t *pending_rules,
+		      bool drain)
+{
+	bool queue_full = *pending_rules == MLX5DR_BWC_MATCHER_REHASH_QUEUE_SZ;
+	bool got_comp = *pending_rules >= MLX5DR_BWC_MATCHER_REHASH_BURST_TH;
+	struct rte_flow_op_result comp[MLX5DR_BWC_MATCHER_REHASH_BURST_TH];
+	int ret;
+	int i;
+
+	/* Check if there are any completions at all */
+	if (!got_comp && !drain)
+		return 0;
+
+	while (queue_full || ((got_comp || drain) && *pending_rules)) {
+		ret = mlx5dr_send_queue_poll(ctx, queue_id, comp,
+					     MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+		if (unlikely(ret < 0)) {
+			DR_LOG(ERR, "Rehash error: polling queue %d returned %d\n",
+			       queue_id, ret);
+			return -EINVAL;
+		}
+
+		if (ret) {
+			(*pending_rules) -= ret;
+			for (i = 0; i < ret; i++) {
+				if (unlikely(comp[i].status != RTE_FLOW_OP_SUCCESS))
+					DR_LOG(ERR,
+					       "Rehash error: polling queue %d returned completion with error\n",
+					       queue_id);
+			}
+			queue_full = false;
+		}
+
+		got_comp = !!ret;
+	}
+
+	return 0;
+}
+
+static void
+mlx5dr_bwc_rule_fill_attr(struct mlx5dr_bwc_matcher *bwc_matcher,
+			  uint16_t bwc_queue_idx,
+			  struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+
+	/* no use of INSERT_BY_INDEX in bwc rule */
+	rule_attr->rule_idx = 0;
+
+	/* notify HW at each rule insertion/deletion */
+	rule_attr->burst = 0;
+
+	/* We don't need user data, but the API requires it to exist */
+	rule_attr->user_data = (void *)0xFACADE;
+
+	rule_attr->queue_id = mlx5dr_bwc_get_queue_id(ctx, bwc_queue_idx);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_alloc(void)
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+
+	bwc_rule = simple_calloc(1, sizeof(*bwc_rule));
+	if (unlikely(!bwc_rule))
+		goto out_err;
+
+	bwc_rule->rule = simple_calloc(1, sizeof(*bwc_rule->rule));
+	if (unlikely(!bwc_rule->rule))
+		goto free_rule;
+
+	return bwc_rule;
+
+free_rule:
+	simple_free(bwc_rule);
+out_err:
+	rte_errno = ENOMEM;
+	return NULL;
+}
+
+static void
+mlx5dr_bwc_rule_free(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	if (likely(bwc_rule->rule))
+		simple_free(bwc_rule->rule);
+	simple_free(bwc_rule);
+}
+
+static void
+mlx5dr_bwc_rule_list_add(struct mlx5dr_bwc_rule *bwc_rule, uint16_t idx)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+
+	rte_atomic_fetch_add_explicit(&bwc_matcher->num_of_rules, 1, rte_memory_order_relaxed);
+	bwc_rule->bwc_queue_idx = idx;
+	LIST_INSERT_HEAD(&bwc_matcher->rules[idx], bwc_rule, next);
+}
+
+static void mlx5dr_bwc_rule_list_remove(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+
+	rte_atomic_fetch_sub_explicit(&bwc_matcher->num_of_rules, 1, rte_memory_order_relaxed);
+	LIST_REMOVE(bwc_rule, next);
+}
+
+static int
+mlx5dr_bwc_rule_destroy_hws_async(struct mlx5dr_bwc_rule *bwc_rule,
+				  struct mlx5dr_rule_attr *attr)
+{
+	return mlx5dr_rule_destroy(bwc_rule->rule, attr);
+}
+
+static int
+mlx5dr_bwc_rule_destroy_hws_sync(struct mlx5dr_bwc_rule *bwc_rule,
+				 struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx;
+	struct rte_flow_op_result completion;
+	int ret;
+
+	ret = mlx5dr_bwc_rule_destroy_hws_async(bwc_rule, rule_attr);
+	if (unlikely(ret))
+		return ret;
+
+	do {
+		ret = mlx5dr_send_queue_poll(ctx, rule_attr->queue_id, &completion, 1);
+	} while (ret != 1);
+
+	if (unlikely(completion.status != RTE_FLOW_OP_SUCCESS ||
+		     (bwc_rule->rule->status != MLX5DR_RULE_STATUS_DELETED &&
+		      bwc_rule->rule->status != MLX5DR_RULE_STATUS_DELETING))) {
+		DR_LOG(ERR, "Failed destroying BWC rule: completion %d, rule status %d",
+		       completion.status, bwc_rule->rule->status);
+		rte_errno = EINVAL;
+		return rte_errno;
+	}
+
+	return 0;
+}
+
+static int mlx5dr_bwc_rule_destroy_hws(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	uint16_t idx = bwc_rule->bwc_queue_idx;
+	struct mlx5dr_rule_attr attr;
+	rte_spinlock_t *queue_lock;
+	int ret;
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, idx, &attr);
+
+	queue_lock = mlx5dr_bwc_get_queue_lock(ctx, idx);
+
+	rte_spinlock_lock(queue_lock);
+
+	ret = mlx5dr_bwc_rule_destroy_hws_sync(bwc_rule, &attr);
+	mlx5dr_bwc_rule_list_remove(bwc_rule);
+
+	rte_spinlock_unlock(queue_lock);
+
+	mlx5dr_bwc_rule_free(bwc_rule);
+
+	return ret;
+}
+
+static int mlx5dr_bwc_rule_destroy_root(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	int ret;
+
+	ret = mlx5dr_rule_destroy_root_no_comp(bwc_rule->rule);
+
+	mlx5dr_bwc_rule_free(bwc_rule);
+
+	return ret;
+}
+
+int mlx5dr_bwc_rule_destroy(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	if (unlikely(mlx5dr_table_is_root(bwc_rule->bwc_matcher->matcher->tbl)))
+		return mlx5dr_bwc_rule_destroy_root(bwc_rule);
+
+	return mlx5dr_bwc_rule_destroy_hws(bwc_rule);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws_async(struct mlx5dr_bwc_matcher *bwc_matcher,
+				 const struct rte_flow_item flow_items[],
+				 uint8_t at_idx,
+				 struct mlx5dr_rule_action rule_actions[],
+				 struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_alloc();
+	if (unlikely(!bwc_rule))
+		return NULL;
+
+	bwc_rule->bwc_matcher = bwc_matcher;
+
+	ret = mlx5dr_rule_create(bwc_matcher->matcher,
+				 0, /* only one match template supported */
+				 flow_items,
+				 at_idx,
+				 rule_actions,
+				 rule_attr,
+				 bwc_rule->rule);
+
+	if (unlikely(ret)) {
+		mlx5dr_bwc_rule_free(bwc_rule);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_root_sync(struct mlx5dr_bwc_matcher *bwc_matcher,
+				 const struct rte_flow_item flow_items[],
+				 uint8_t num_actions,
+				 struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_alloc();
+	if (unlikely(!bwc_rule)) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	bwc_rule->bwc_matcher = bwc_matcher;
+	bwc_rule->rule->matcher = bwc_matcher->matcher;
+
+	ret = mlx5dr_rule_create_root_no_comp(bwc_rule->rule,
+					      flow_items,
+					      num_actions,
+					      rule_actions);
+	if (unlikely(ret)) {
+		mlx5dr_bwc_rule_free(bwc_rule);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws_sync(struct mlx5dr_bwc_matcher *bwc_matcher,
+				const struct rte_flow_item flow_items[],
+				uint8_t at_idx,
+				struct mlx5dr_rule_action rule_actions[],
+				struct mlx5dr_rule_attr *rule_attr)
+
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	struct rte_flow_op_result completion;
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_async(bwc_matcher, flow_items,
+						    at_idx, rule_actions,
+						    rule_attr);
+	if (unlikely(!bwc_rule))
+		return NULL;
+
+	do {
+		ret = mlx5dr_send_queue_poll(ctx, rule_attr->queue_id, &completion, 1);
+	} while (ret != 1);
+
+	if (unlikely(completion.status != RTE_FLOW_OP_SUCCESS ||
+		     (bwc_rule->rule->status != MLX5DR_RULE_STATUS_CREATED &&
+		      bwc_rule->rule->status != MLX5DR_RULE_STATUS_CREATING))) {
+		DR_LOG(ERR, "Failed creating BWC rule: completion %d, rule status %d",
+		       completion.status, bwc_rule->rule->status);
+		mlx5dr_bwc_rule_free(bwc_rule);
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static bool
+mlx5dr_bwc_matcher_size_maxed_out(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps;
+
+	return bwc_matcher->size_log + MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH >=
+	       caps->ste_alloc_log_max - 1;
+}
+
+static bool
+mlx5dr_bwc_matcher_rehash_size_needed(struct mlx5dr_bwc_matcher *bwc_matcher,
+				      uint32_t num_of_rules)
+{
+	/* size-based rehash for root table is kernel's responsibility */
+	if (unlikely(mlx5dr_table_is_root(bwc_matcher->matcher->tbl)))
+		return false;
+
+	if (unlikely(mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher)))
+		return false;
+
+	if (unlikely((num_of_rules * 100 / MLX5DR_BWC_MATCHER_REHASH_PERCENT_TH) >=
+		     (1UL << bwc_matcher->size_log)))
+		return true;
+
+	return false;
+}
+
+static void
+mlx5dr_bwc_rule_actions_to_action_types(struct mlx5dr_rule_action rule_actions[],
+					enum mlx5dr_action_type action_types[])
+{
+	int i = 0;
+
+	for (i = 0;
+	     rule_actions[i].action && (rule_actions[i].action->type != MLX5DR_ACTION_TYP_LAST);
+	     i++) {
+		action_types[i] = (enum mlx5dr_action_type)rule_actions[i].action->type;
+	}
+
+	action_types[i] = MLX5DR_ACTION_TYP_LAST;
+}
+
+static int
+mlx5dr_bwc_rule_actions_num(struct mlx5dr_rule_action rule_actions[])
+{
+	int i = 0;
+
+	while (rule_actions[i].action &&
+	       (rule_actions[i].action->type != MLX5DR_ACTION_TYP_LAST))
+		i++;
+
+	return i;
+}
+
+static int
+mlx5dr_bwc_matcher_extend_at(struct mlx5dr_bwc_matcher *bwc_matcher,
+			     struct mlx5dr_rule_action rule_actions[])
+{
+	enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS];
+
+	mlx5dr_bwc_rule_actions_to_action_types(rule_actions, action_types);
+
+	bwc_matcher->at[bwc_matcher->num_of_at] =
+		mlx5dr_action_template_create(action_types, 0);
+
+	if (unlikely(!bwc_matcher->at[bwc_matcher->num_of_at])) {
+		rte_errno = ENOMEM;
+		return rte_errno;
+	}
+
+	bwc_matcher->num_of_at++;
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_extend_size(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps;
+
+	if (unlikely(mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher))) {
+		DR_LOG(ERR, "Can't resize matcher: depth exceeds limit %d",
+		       caps->rtc_log_depth_max);
+		return -ENOMEM;
+	}
+
+	bwc_matcher->size_log =
+		RTE_MIN(bwc_matcher->size_log + MLX5DR_BWC_MATCHER_SIZE_LOG_STEP,
+			caps->ste_alloc_log_max - MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_find_at(struct mlx5dr_bwc_matcher *bwc_matcher,
+			   struct mlx5dr_rule_action rule_actions[])
+{
+	enum mlx5dr_action_type *action_type_arr;
+	int i, j;
+
+	/* start from index 1 - first action template is a dummy */
+	for (i = 1; i < bwc_matcher->num_of_at; i++) {
+		j = 0;
+		action_type_arr = bwc_matcher->at[i]->action_type_arr;
+
+		while (rule_actions[j].action &&
+		       rule_actions[j].action->type != MLX5DR_ACTION_TYP_LAST) {
+			if (action_type_arr[j] != rule_actions[j].action->type)
+				break;
+			j++;
+		}
+
+		if (action_type_arr[j] == MLX5DR_ACTION_TYP_LAST &&
+		    (!rule_actions[j].action ||
+		     rule_actions[j].action->type == MLX5DR_ACTION_TYP_LAST))
+			return i;
+	}
+
+	return -1;
+}
+
+static int
+mlx5dr_bwc_matcher_move_all(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	struct mlx5dr_bwc_rule **bwc_rules;
+	struct mlx5dr_rule_attr rule_attr;
+	uint32_t *pending_rules;
+	bool all_done;
+	int i, j, ret;
+
+	if (mlx5dr_table_is_root(bwc_matcher->matcher->tbl)) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, 0, &rule_attr);
+
+	pending_rules = simple_calloc(bwc_queues, sizeof(*pending_rules));
+	if (!pending_rules) {
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+
+	bwc_rules = simple_calloc(bwc_queues, sizeof(*bwc_rules));
+	if (!bwc_rules) {
+		rte_errno = ENOMEM;
+		goto free_pending_rules;
+	}
+
+	for (i = 0; i < bwc_queues; i++) {
+		if (LIST_EMPTY(&bwc_matcher->rules[i]))
+			bwc_rules[i] = NULL;
+		else
+			bwc_rules[i] = LIST_FIRST(&bwc_matcher->rules[i]);
+	}
+
+	do {
+		all_done = true;
+
+		for (i = 0; i < bwc_queues; i++) {
+			rule_attr.queue_id = mlx5dr_bwc_get_queue_id(ctx, i);
+
+			for (j = 0; j < MLX5DR_BWC_MATCHER_REHASH_BURST_TH && bwc_rules[i]; j++) {
+				rule_attr.burst = !!((j + 1) % MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+				ret = mlx5dr_matcher_resize_rule_move(bwc_matcher->matcher,
+								      bwc_rules[i]->rule,
+								      &rule_attr);
+				if (ret) {
+					DR_LOG(ERR, "Moving BWC rule failed during rehash - %d",
+					       ret);
+					rte_errno = ENOMEM;
+					goto free_bwc_rules;
+				}
+
+				all_done = false;
+				pending_rules[i]++;
+				bwc_rules[i] = LIST_NEXT(bwc_rules[i], next);
+
+				mlx5dr_bwc_queue_poll(ctx, rule_attr.queue_id,
+						      &pending_rules[i], false);
+			}
+		}
+	} while (!all_done);
+
+	/* drain all the bwc queues */
+	for (i = 0; i < bwc_queues; i++) {
+		if (pending_rules[i]) {
+			uint16_t queue_id = mlx5dr_bwc_get_queue_id(ctx, i);
+			mlx5dr_send_engine_flush_queue(&ctx->send_queue[queue_id]);
+			mlx5dr_bwc_queue_poll(ctx, queue_id,
+					      &pending_rules[i], true);
+		}
+	}
+
+	rte_errno = 0;
+
+free_bwc_rules:
+	simple_free(bwc_rules);
+free_pending_rules:
+	simple_free(pending_rules);
+
+	return -rte_errno;
+}
+
+static int
+mlx5dr_bwc_matcher_move(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_matcher_attr matcher_attr = {0};
+	struct mlx5dr_matcher *old_matcher;
+	struct mlx5dr_matcher *new_matcher;
+	int ret;
+
+	mlx5dr_bwc_matcher_init_attr(&matcher_attr,
+				     bwc_matcher->priority,
+				     bwc_matcher->size_log,
+				     mlx5dr_table_is_root(bwc_matcher->matcher->tbl));
+
+	old_matcher = bwc_matcher->matcher;
+	new_matcher = mlx5dr_matcher_create(old_matcher->tbl,
+					    &bwc_matcher->mt, 1,
+					    bwc_matcher->at,
+					    bwc_matcher->num_of_at,
+					    &matcher_attr);
+	if (!new_matcher) {
+		DR_LOG(ERR, "Rehash error: matcher creation failed");
+		return -ENOMEM;
+	}
+
+	ret = mlx5dr_matcher_resize_set_target(old_matcher, new_matcher);
+	if (ret) {
+		DR_LOG(ERR, "Rehash error: failed setting resize target");
+		return ret;
+	}
+
+	ret = mlx5dr_bwc_matcher_move_all(bwc_matcher);
+	if (ret) {
+		DR_LOG(ERR, "Rehash error: moving rules failed");
+		return -ENOMEM;
+	}
+
+	bwc_matcher->matcher = new_matcher;
+	mlx5dr_matcher_destroy(old_matcher);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_rehash(struct mlx5dr_bwc_matcher *bwc_matcher, bool rehash_size)
+{
+	uint32_t num_of_rules;
+	int ret;
+
+	/* If the current matcher size is already at its max size, we can't
+	 * do the rehash. Skip it and try adding the rule again - perhaps
+	 * there was some change.
+	 */
+	if (mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher))
+		return 0;
+
+	/* It is possible that other rule has already performed rehash.
+	 * Need to check again if we really need rehash.
+	 * If the reason for rehash was size, but not any more - skip rehash.
+	 */
+	num_of_rules = rte_atomic_load_explicit(&bwc_matcher->num_of_rules,
+						rte_memory_order_relaxed);
+	if (rehash_size &&
+	    !mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules))
+		return 0;
+
+	/* Now we're done all the checking - do the rehash:
+	 *  - extend match RTC size
+	 *  - create new matcher
+	 *  - move all the rules to the new matcher
+	 *  - destroy the old matcher
+	 */
+
+	ret = mlx5dr_bwc_matcher_extend_size(bwc_matcher);
+	if (ret)
+		return ret;
+
+	return mlx5dr_bwc_matcher_move(bwc_matcher);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_root(struct mlx5dr_bwc_matcher *bwc_matcher,
+			    const struct rte_flow_item flow_items[],
+			    struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+
+	bwc_rule = mlx5dr_bwc_rule_create_root_sync(bwc_matcher,
+						    flow_items,
+						    mlx5dr_bwc_rule_actions_num(rule_actions),
+						    rule_actions);
+
+	if (unlikely(!bwc_rule))
+		DR_LOG(ERR, "BWC rule: failed creating rule on root tbl");
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws(struct mlx5dr_bwc_matcher *bwc_matcher,
+			   const struct rte_flow_item flow_items[],
+			   struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	struct mlx5dr_bwc_rule *bwc_rule = NULL;
+	struct mlx5dr_rule_attr rule_attr;
+	rte_spinlock_t *queue_lock;
+	bool rehash_size = false;
+	uint32_t num_of_rules;
+	uint16_t idx;
+	int at_idx;
+	int ret;
+
+	idx = mlx5dr_bwc_gen_queue_idx(ctx);
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, idx, &rule_attr);
+
+	queue_lock = mlx5dr_bwc_get_queue_lock(ctx, idx);
+
+	rte_spinlock_lock(queue_lock);
+
+	/* check if rehash needed due to missing action template */
+	at_idx = mlx5dr_bwc_matcher_find_at(bwc_matcher, rule_actions);
+	if (at_idx < 0) {
+		/* we need to extend BWC matcher action templates array */
+		rte_spinlock_unlock(queue_lock);
+		mlx5dr_bwc_lock_all_queues(ctx);
+
+		ret = mlx5dr_bwc_matcher_extend_at(bwc_matcher, rule_actions);
+		if (unlikely(ret)) {
+			mlx5dr_bwc_unlock_all_queues(ctx);
+			rte_errno = EINVAL;
+			DR_LOG(ERR, "BWC rule: failed extending action template - %d", ret);
+			return NULL;
+		}
+
+		/* action templates array was extended, we need the last idx */
+		at_idx = bwc_matcher->num_of_at - 1;
+
+		ret = mlx5dr_matcher_attach_at(bwc_matcher->matcher,
+					       bwc_matcher->at[at_idx]);
+		if (unlikely(ret)) {
+			mlx5dr_action_template_destroy(bwc_matcher->at[at_idx]);
+			bwc_matcher->at[at_idx] = NULL;
+			bwc_matcher->num_of_at--;
+
+			mlx5dr_bwc_unlock_all_queues(ctx);
+			DR_LOG(ERR, "BWC rule: failed attaching action template - %d", ret);
+			return NULL;
+		}
+
+		mlx5dr_bwc_unlock_all_queues(ctx);
+		rte_spinlock_lock(queue_lock);
+	}
+
+	/* check if number of rules require rehash */
+	num_of_rules = rte_atomic_load_explicit(&bwc_matcher->num_of_rules,
+						rte_memory_order_relaxed);
+	if (mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules)) {
+		rehash_size = true;
+		goto rehash;
+	}
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_sync(bwc_matcher,
+						   flow_items,
+						   at_idx,
+						   rule_actions,
+						   &rule_attr);
+
+	if (likely(bwc_rule)) {
+		mlx5dr_bwc_rule_list_add(bwc_rule, idx);
+		rte_spinlock_unlock(queue_lock);
+		return bwc_rule; /* rule inserted successfully */
+	}
+
+	/* At this point the rule wasn't added.
+	 * It could be because there was collision, or some other problem.
+	 * If we don't dive deeper than API, the only thing we know is that
+	 * the status of completion is RTE_FLOW_OP_ERROR.
+	 * Try rehash and insert rule again - last chance.
+	 */
+
+rehash:
+	rte_spinlock_unlock(queue_lock);
+
+	mlx5dr_bwc_lock_all_queues(ctx);
+	ret = mlx5dr_bwc_matcher_rehash(bwc_matcher, rehash_size);
+	mlx5dr_bwc_unlock_all_queues(ctx);
+
+	if (ret) {
+		DR_LOG(ERR, "BWC rule insertion: rehash failed - %d", ret);
+		return NULL;
+	}
+
+	/* Rehash done, but we still have that pesky rule to add */
+	rte_spinlock_lock(queue_lock);
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_sync(bwc_matcher,
+						   flow_items,
+						   at_idx,
+						   rule_actions,
+						   &rule_attr);
+
+	if (unlikely(!bwc_rule)) {
+		rte_spinlock_unlock(queue_lock);
+		DR_LOG(ERR, "BWC rule insertion failed");
+		return NULL;
+	}
+
+	mlx5dr_bwc_rule_list_add(bwc_rule, idx);
+	rte_spinlock_unlock(queue_lock);
+
+	return bwc_rule;
+}
+
+struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create(struct mlx5dr_bwc_matcher *bwc_matcher,
+		       const struct rte_flow_item flow_items[],
+		       struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+
+	if (unlikely(!mlx5dr_context_bwc_supported(ctx))) {
+		rte_errno = EINVAL;
+		DR_LOG(ERR, "BWC rule: Context created w/o BWC API compatibility");
+		return NULL;
+	}
+
+	if (unlikely(mlx5dr_table_is_root(bwc_matcher->matcher->tbl)))
+		return mlx5dr_bwc_rule_create_root(bwc_matcher,
+						   flow_items,
+						   rule_actions);
+
+	return mlx5dr_bwc_rule_create_hws(bwc_matcher,
+					  flow_items,
+					  rule_actions);
+}
diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.h b/drivers/net/mlx5/hws/mlx5dr_bwc.h
new file mode 100644
index 0000000000..648443861b
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5DR_BWC_H_
+#define MLX5DR_BWC_H_
+
+#define MLX5DR_BWC_MATCHER_INIT_SIZE_LOG 1
+#define MLX5DR_BWC_MATCHER_SIZE_LOG_STEP 1
+#define MLX5DR_BWC_MATCHER_REHASH_PERCENT_TH 70
+#define MLX5DR_BWC_MATCHER_REHASH_BURST_TH 32
+#define MLX5DR_BWC_MATCHER_REHASH_QUEUE_SZ 256
+#define MLX5DR_BWC_MATCHER_ATTACH_AT_NUM 255
+
+struct mlx5dr_bwc_matcher {
+	struct mlx5dr_matcher *matcher;
+	struct mlx5dr_match_template *mt;
+	struct mlx5dr_action_template *at[MLX5DR_BWC_MATCHER_ATTACH_AT_NUM];
+	uint8_t num_of_at;
+	uint32_t priority;
+	uint8_t size_log;
+	RTE_ATOMIC(uint32_t)num_of_rules; /* atomically accessed */
+	LIST_HEAD(rule_head, mlx5dr_bwc_rule) * rules;
+};
+
+struct mlx5dr_bwc_rule {
+	struct mlx5dr_bwc_matcher *bwc_matcher;
+	struct mlx5dr_rule *rule;
+	struct rte_flow_item *flow_items;
+	uint16_t bwc_queue_idx;
+	LIST_ENTRY(mlx5dr_bwc_rule) next;
+};
+
+#endif /* MLX5DR_BWC_H_ */
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.c b/drivers/net/mlx5/hws/mlx5dr_context.c
index 7f120b3b1b..db5e72927a 100644
--- a/drivers/net/mlx5/hws/mlx5dr_context.c
+++ b/drivers/net/mlx5/hws/mlx5dr_context.c
@@ -176,6 +176,9 @@ static int mlx5dr_context_init_hws(struct mlx5dr_context *ctx,
 	if (ret)
 		goto uninit_pd;
 
+	if (attr->bwc)
+		ctx->flags |= MLX5DR_CONTEXT_FLAG_BWC_SUPPORT;
+
 	ret = mlx5dr_send_queues_open(ctx, attr->queues, attr->queue_size);
 	if (ret)
 		goto pools_uninit;
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.h b/drivers/net/mlx5/hws/mlx5dr_context.h
index f476c2308c..7678232ecc 100644
--- a/drivers/net/mlx5/hws/mlx5dr_context.h
+++ b/drivers/net/mlx5/hws/mlx5dr_context.h
@@ -8,6 +8,7 @@
 enum mlx5dr_context_flags {
 	MLX5DR_CONTEXT_FLAG_HWS_SUPPORT = 1 << 0,
 	MLX5DR_CONTEXT_FLAG_PRIVATE_PD = 1 << 1,
+	MLX5DR_CONTEXT_FLAG_BWC_SUPPORT = 1 << 2,
 };
 
 enum mlx5dr_context_shared_stc_type {
@@ -44,6 +45,7 @@ struct mlx5dr_context {
 	enum mlx5dr_context_flags flags;
 	struct mlx5dr_send_engine *send_queue;
 	size_t queues;
+	rte_spinlock_t *bwc_send_queue_locks;
 	LIST_HEAD(table_head, mlx5dr_table) head;
 };
 
@@ -52,6 +54,11 @@ static inline bool mlx5dr_context_shared_gvmi_used(struct mlx5dr_context *ctx)
 	return ctx->local_ibv_ctx ? true : false;
 }
 
+static inline bool mlx5dr_context_bwc_supported(struct mlx5dr_context *ctx)
+{
+	return ctx->flags & MLX5DR_CONTEXT_FLAG_BWC_SUPPORT;
+}
+
 static inline struct ibv_context *
 mlx5dr_context_get_local_ibv(struct mlx5dr_context *ctx)
 {
diff --git a/drivers/net/mlx5/hws/mlx5dr_internal.h b/drivers/net/mlx5/hws/mlx5dr_internal.h
index b9efdc4a9a..2abc516b5e 100644
--- a/drivers/net/mlx5/hws/mlx5dr_internal.h
+++ b/drivers/net/mlx5/hws/mlx5dr_internal.h
@@ -39,6 +39,7 @@
 #include "mlx5dr_debug.h"
 #include "mlx5dr_pat_arg.h"
 #include "mlx5dr_crc32.h"
+#include "mlx5dr_bwc.h"
 
 #define W_SIZE		2
 #define DW_SIZE		4
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c b/drivers/net/mlx5/hws/mlx5dr_send.c
index 2942668e76..0120f03a48 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.c
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -971,10 +971,42 @@ static void __mlx5dr_send_queues_close(struct mlx5dr_context *ctx, uint16_t queu
 		mlx5dr_send_queue_close(&ctx->send_queue[queues]);
 }
 
+static int mlx5dr_bwc_send_queues_init(struct mlx5dr_context *ctx)
+{
+	int bwc_queues = ctx->queues - 1;
+	int i;
+
+	if (!mlx5dr_context_bwc_supported(ctx))
+		return 0;
+
+	ctx->queues += bwc_queues;
+
+	ctx->bwc_send_queue_locks = simple_calloc(bwc_queues,
+						  sizeof(*ctx->bwc_send_queue_locks));
+	if (!ctx->bwc_send_queue_locks) {
+		rte_errno = ENOMEM;
+		return rte_errno;
+	}
+
+	for (i = 0; i < bwc_queues; i++)
+		rte_spinlock_init(&ctx->bwc_send_queue_locks[i]);
+
+	return 0;
+}
+
+static void mlx5dr_send_queues_bwc_locks_destroy(struct mlx5dr_context *ctx)
+{
+	if (!mlx5dr_context_bwc_supported(ctx))
+		return;
+
+	simple_free(ctx->bwc_send_queue_locks);
+}
+
 void mlx5dr_send_queues_close(struct mlx5dr_context *ctx)
 {
 	__mlx5dr_send_queues_close(ctx, ctx->queues);
 	simple_free(ctx->send_queue);
+	mlx5dr_send_queues_bwc_locks_destroy(ctx);
 }
 
 int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
@@ -987,10 +1019,16 @@ int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
 	/* Open one extra queue for control path */
 	ctx->queues = queues + 1;
 
+	/* open a separate set of queues and locks for bwc API */
+	err = mlx5dr_bwc_send_queues_init(ctx);
+	if (err)
+		return err;
+
 	ctx->send_queue = simple_calloc(ctx->queues, sizeof(*ctx->send_queue));
 	if (!ctx->send_queue) {
 		rte_errno = ENOMEM;
-		return rte_errno;
+		err = rte_errno;
+		goto free_bwc_locks;
 	}
 
 	for (i = 0; i < ctx->queues; i++) {
@@ -1006,6 +1044,9 @@ int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
 
 	simple_free(ctx->send_queue);
 
+free_bwc_locks:
+	mlx5dr_send_queues_bwc_locks_destroy(ctx);
+
 	return err;
 }
 
-- 
2.25.1

[PATCH v4 0/3] HWS non-template support

[Maayan Kashani]

Initial coding to support non-template in mlxdr layer.

Yevgeny Kliteynik (3):
  net/mlx5/hws: split the root rule creation and destruction
  net/mlx5/hws: add support for action type LAST
  net/mlx5/hws: add support for backward-compatible API

 drivers/net/mlx5/hws/meson.build       |   1 +
 drivers/net/mlx5/hws/mlx5dr.h          |  87 +++
 drivers/net/mlx5/hws/mlx5dr_action.c   |   9 +
 drivers/net/mlx5/hws/mlx5dr_bwc.c      | 898 +++++++++++++++++++++++++
 drivers/net/mlx5/hws/mlx5dr_bwc.h      |  34 +
 drivers/net/mlx5/hws/mlx5dr_context.c  |   3 +
 drivers/net/mlx5/hws/mlx5dr_context.h  |   7 +
 drivers/net/mlx5/hws/mlx5dr_internal.h |   1 +
 drivers/net/mlx5/hws/mlx5dr_rule.c     |  49 +-
 drivers/net/mlx5/hws/mlx5dr_rule.h     |   7 +
 drivers/net/mlx5/hws/mlx5dr_send.c     |  43 +-
 11 files changed, 1124 insertions(+), 15 deletions(-)
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.c
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.h

-- 
2.21.0

[PATCH v4 1/3] net/mlx5/hws: split the root rule creation and destruction

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

Split the root rule creation/destruction into two stages:
 - do the job (create/destroy rule)
 - generate completion

Completion generation is required for the usual HWS API.
The create/destroy functions that don't generate completion
are exposed in header file and will be used by the coming
backward-compatible API.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_rule.c | 49 +++++++++++++++++++++---------
 drivers/net/mlx5/hws/mlx5dr_rule.h |  7 +++++
 2 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c b/drivers/net/mlx5/hws/mlx5dr_rule.c
index 171a0bff38..550f00a4c1 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.c
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -680,15 +680,12 @@ static int mlx5dr_rule_destroy_hws(struct mlx5dr_rule *rule,
 	return 0;
 }
 
-static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
-				   struct mlx5dr_rule_attr *rule_attr,
-				   const struct rte_flow_item items[],
-				   uint8_t at_idx,
-				   struct mlx5dr_rule_action rule_actions[])
+int mlx5dr_rule_create_root_no_comp(struct mlx5dr_rule *rule,
+				    const struct rte_flow_item items[],
+				    uint8_t num_actions,
+				    struct mlx5dr_rule_action rule_actions[])
 {
 	struct mlx5dv_flow_matcher *dv_matcher = rule->matcher->dv_matcher;
-	uint8_t num_actions = rule->matcher->at[at_idx].num_actions;
-	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
 	struct mlx5dv_flow_match_parameters *value;
 	struct mlx5_flow_attr flow_attr = {0};
 	struct mlx5dv_flow_action_attr *attr;
@@ -750,9 +747,6 @@ static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
 						    num_actions,
 						    attr);
 
-	mlx5dr_rule_gen_comp(&ctx->send_queue[rule_attr->queue_id], rule, !rule->flow,
-			     rule_attr->user_data, MLX5DR_RULE_STATUS_CREATED);
-
 	simple_free(value);
 	simple_free(attr);
 
@@ -766,14 +760,41 @@ static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
 	return rte_errno;
 }
 
+static int mlx5dr_rule_create_root(struct mlx5dr_rule *rule,
+				   struct mlx5dr_rule_attr *rule_attr,
+				   const struct rte_flow_item items[],
+				   uint8_t num_actions,
+				   struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
+	int ret;
+
+	ret = mlx5dr_rule_create_root_no_comp(rule, items,
+					      num_actions, rule_actions);
+	if (ret)
+		return rte_errno;
+
+	mlx5dr_rule_gen_comp(&ctx->send_queue[rule_attr->queue_id], rule, !rule->flow,
+			     rule_attr->user_data, MLX5DR_RULE_STATUS_CREATED);
+
+	return 0;
+}
+
+int mlx5dr_rule_destroy_root_no_comp(struct mlx5dr_rule *rule)
+{
+	if (rule->flow)
+		return ibv_destroy_flow(rule->flow);
+
+	return 0;
+}
+
 static int mlx5dr_rule_destroy_root(struct mlx5dr_rule *rule,
 				    struct mlx5dr_rule_attr *attr)
 {
 	struct mlx5dr_context *ctx = rule->matcher->tbl->ctx;
-	int err = 0;
+	int err;
 
-	if (rule->flow)
-		err = ibv_destroy_flow(rule->flow);
+	err = mlx5dr_rule_destroy_root_no_comp(rule);
 
 	mlx5dr_rule_gen_comp(&ctx->send_queue[attr->queue_id], rule, err,
 			     attr->user_data, MLX5DR_RULE_STATUS_DELETED);
@@ -970,7 +991,7 @@ int mlx5dr_rule_create(struct mlx5dr_matcher *matcher,
 		ret = mlx5dr_rule_create_root(rule_handle,
 					      attr,
 					      items,
-					      at_idx,
+					      matcher->at[at_idx].num_actions,
 					      rule_actions);
 	else
 		ret = mlx5dr_rule_create_hws(rule_handle,
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.h b/drivers/net/mlx5/hws/mlx5dr_rule.h
index dffaec1c0f..bc542eb543 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.h
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.h
@@ -74,4 +74,11 @@ int mlx5dr_rule_move_hws_add(struct mlx5dr_rule *rule,
 
 bool mlx5dr_rule_move_in_progress(struct mlx5dr_rule *rule);
 
+int mlx5dr_rule_create_root_no_comp(struct mlx5dr_rule *rule,
+				    const struct rte_flow_item items[],
+				    uint8_t num_actions,
+				    struct mlx5dr_rule_action rule_actions[]);
+
+int mlx5dr_rule_destroy_root_no_comp(struct mlx5dr_rule *rule);
+
 #endif /* MLX5DR_RULE_H_ */
-- 
2.21.0

[PATCH v4 2/3] net/mlx5/hws: add support for action type LAST

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

Add support for MLX5DR_ACTION_TYP_LAST action type.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr.h        | 12 ++++++++++++
 drivers/net/mlx5/hws/mlx5dr_action.c |  9 +++++++++
 2 files changed, 21 insertions(+)

diff --git a/drivers/net/mlx5/hws/mlx5dr.h b/drivers/net/mlx5/hws/mlx5dr.h
index 86cd4f1db5..5ceb1a7b4b 100644
--- a/drivers/net/mlx5/hws/mlx5dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -677,6 +677,18 @@ struct mlx5dr_action *
 mlx5dr_action_create_tag(struct mlx5dr_context *ctx,
 			 uint32_t flags);
 
+/* Create direct rule LAST action.
+ *
+ * @param[in] ctx
+ *	The context in which the new action will be created.
+ * @param[in] flags
+ *	Action creation flags. (enum mlx5dr_action_flags)
+ * @return pointer to mlx5dr_action on success NULL otherwise.
+ */
+struct mlx5dr_action *
+mlx5dr_action_create_last(struct mlx5dr_context *ctx,
+			  uint32_t flags);
+
 /* Create direct rule counter action.
  *
  * @param[in] ctx
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.c b/drivers/net/mlx5/hws/mlx5dr_action.c
index 562fb5cbb4..90f2f17bd0 100644
--- a/drivers/net/mlx5/hws/mlx5dr_action.c
+++ b/drivers/net/mlx5/hws/mlx5dr_action.c
@@ -1319,6 +1319,13 @@ mlx5dr_action_create_tag(struct mlx5dr_context *ctx,
 	return NULL;
 }
 
+struct mlx5dr_action *
+mlx5dr_action_create_last(struct mlx5dr_context *ctx,
+			  uint32_t flags)
+{
+	return mlx5dr_action_create_generic(ctx, flags, MLX5DR_ACTION_TYP_LAST);
+}
+
 static struct mlx5dr_action *
 mlx5dr_action_create_aso(struct mlx5dr_context *ctx,
 			 enum mlx5dr_action_type action_type,
@@ -3005,6 +3012,8 @@ static void mlx5dr_action_destroy_hws(struct mlx5dr_action *action)
 		for (i = 0; i < MLX5DR_ACTION_NAT64_STAGES; i++)
 			mlx5dr_action_destroy(action->nat64.stages[i]);
 		break;
+	case MLX5DR_ACTION_TYP_LAST:
+		break;
 	}
 }
 
-- 
2.21.0

[PATCH v4 3/3] net/mlx5/hws: add support for backward-compatible API

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

HWS API is very different from SWS and has limitations:
 - Queue based async insertion/deletion
 - Doesn’t handle complex rules
 - Requires the user to specify fixed matcher size – no rehash
 - Requires the user to specify action combination on matcher creation
 - Matching fields are passed using items and not FW/PRM format

These problems need to be solved in order to support HWS with
the old rte_flow API.

This patch adds backward-compatible (BWC) HWS API to allow for the
use of HWS in rte_flow and support the existing template-based API
alongside with the rte_flow API.

The new BWC API comprises mainly of the following four functions:
 - mlx5dr_bwc_matcher_create
 - mlx5dr_bwc_matcher_destroy
 - mlx5dr_bwc_rule_create
 - mlx5dr_bwc_rule_destroy

Notes:
 - To enable BWC API support, need to turn on 'bwc' flag in context
   attributes when calling mlx5dr_context_open()
 - BWC functions use their own queues
 - BWC functions provide synchronous API - they add/remove rule and
   poll for completion
 - BWC functions handle all the required matchers and rules allocations
 - For non-root tables, BWC functions use matcher resize and matcher
   attach action template features

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
 drivers/net/mlx5/hws/meson.build       |   1 +
 drivers/net/mlx5/hws/mlx5dr.h          |  75 +++
 drivers/net/mlx5/hws/mlx5dr_bwc.c      | 898 +++++++++++++++++++++++++
 drivers/net/mlx5/hws/mlx5dr_bwc.h      |  34 +
 drivers/net/mlx5/hws/mlx5dr_context.c  |   3 +
 drivers/net/mlx5/hws/mlx5dr_context.h  |   7 +
 drivers/net/mlx5/hws/mlx5dr_internal.h |   1 +
 drivers/net/mlx5/hws/mlx5dr_send.c     |  43 +-
 8 files changed, 1061 insertions(+), 1 deletion(-)
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.c
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.h

diff --git a/drivers/net/mlx5/hws/meson.build b/drivers/net/mlx5/hws/meson.build
index bbcc628557..859a87fb85 100644
--- a/drivers/net/mlx5/hws/meson.build
+++ b/drivers/net/mlx5/hws/meson.build
@@ -20,4 +20,5 @@ sources += files(
         'mlx5dr_debug.c',
         'mlx5dr_pat_arg.c',
         'mlx5dr_crc32.c',
+        'mlx5dr_bwc.c',
 )
diff --git a/drivers/net/mlx5/hws/mlx5dr.h b/drivers/net/mlx5/hws/mlx5dr.h
index 5ceb1a7b4b..0fe39e9c76 100644
--- a/drivers/net/mlx5/hws/mlx5dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -106,6 +106,7 @@ struct mlx5dr_context_attr {
 	struct ibv_pd *pd;
 	/* Optional other ctx for resources allocation, all objects will be created on it */
 	struct ibv_context *shared_ibv_ctx;
+	bool bwc; /* add support for backward compatible API*/
 };
 
 struct mlx5dr_table_attr {
@@ -992,4 +993,78 @@ int mlx5dr_crc_encap_entropy_hash_calc(struct mlx5dr_context *ctx,
 				       uint8_t entropy_res[],
 				       enum mlx5dr_crc_encap_entropy_hash_size res_size);
 
+struct mlx5dr_bwc_matcher;
+struct mlx5dr_bwc_rule;
+
+/* Create a new BWC direct rule matcher.
+ * This function does the following:
+ *   - creates match template based on flow items
+ *   - creates an empty action template
+ *   - creates a usual mlx5dr_matcher with these mt and at, setting
+ *     its size to minimal
+ * Notes:
+ *   - table->ctx must have BWC support
+ *   - complex rules are not supported
+ *
+ * @param[in] table
+ *	The table in which the new matcher will be opened
+ * @param[in] priority
+ *	Priority for this BWC matcher
+ * @param[in] flow_items
+ *	Array of flow items that serve as basis for match and action templates
+ * @return pointer to mlx5dr_bwc_matcher on success or NULL otherwise.
+ */
+struct mlx5dr_bwc_matcher *
+mlx5dr_bwc_matcher_create(struct mlx5dr_table *table,
+			  uint32_t priority,
+			  const struct rte_flow_item flow_items[]);
+
+/* Destroy BWC direct rule matcher.
+ *
+ * @param[in] bwc_matcher
+ *	Matcher to destroy
+ * @return zero on success, non zero otherwise
+ */
+int mlx5dr_bwc_matcher_destroy(struct mlx5dr_bwc_matcher *bwc_matcher);
+
+/* Create a new BWC rule.
+ * Unlike the usual rule creation function, this one is blocking: when the
+ * function returns, the rule is written to its place (no need to poll).
+ * This function does the following:
+ *   - finds matching action template based on the provided rule_actions, or
+ *     creates new action template if matching action template doesn't exist
+ *   - updates corresponding BWC matcher stats
+ *   - if needed, the function performs rehash:
+ *       - creates a new matcher based on mt, at, new_sz
+ *       - moves all the existing matcher rules to the new matcher
+ *       - removes the old matcher
+ *   - inserts new rule
+ *   - polls till completion is received
+ * Notes:
+ *   - matcher->tbl->ctx must have BWC support
+ *   - separate BWC ctx queues are used
+ *
+ * @param[in] bwc_matcher
+ *	The BWC matcher in which the new rule will be created.
+ * @param[in] flow_items
+ *	Flow items to be used for the value matching
+ * @param[in] rule_actions
+ *	Rule action to be executed on match
+ * @param[in, out] rule_handle
+ *	A valid rule handle. The handle doesn't require any initialization
+ * @return valid BWC rule handle on success, NULL otherwise
+ */
+struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create(struct mlx5dr_bwc_matcher *bwc_matcher,
+		       const struct rte_flow_item flow_items[],
+		       struct mlx5dr_rule_action rule_actions[]);
+
+/* Destroy BWC direct rule.
+ *
+ * @param[in] bwc_rule
+ *	Rule to destroy
+ * @return zero on success, non zero otherwise
+ */
+int mlx5dr_bwc_rule_destroy(struct mlx5dr_bwc_rule *bwc_rule);
+
 #endif
diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.c b/drivers/net/mlx5/hws/mlx5dr_bwc.c
new file mode 100644
index 0000000000..eef3053ee0
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.c
@@ -0,0 +1,898 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static uint16_t mlx5dr_bwc_queues(struct mlx5dr_context *ctx)
+{
+	return (ctx->queues - 1) / 2;
+}
+
+static uint16_t
+mlx5dr_bwc_gen_queue_idx(struct mlx5dr_context *ctx)
+{
+	/* assign random queue */
+	return rand() % mlx5dr_bwc_queues(ctx);
+}
+
+static uint16_t
+mlx5dr_bwc_get_queue_id(struct mlx5dr_context *ctx, uint16_t idx)
+{
+	return idx + mlx5dr_bwc_queues(ctx);
+}
+
+static rte_spinlock_t *
+mlx5dr_bwc_get_queue_lock(struct mlx5dr_context *ctx, uint16_t idx)
+{
+	return &ctx->bwc_send_queue_locks[idx];
+}
+
+static void mlx5dr_bwc_lock_all_queues(struct mlx5dr_context *ctx)
+{
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	rte_spinlock_t *queue_lock;
+	int i;
+
+	for (i = 0; i < bwc_queues; i++) {
+		queue_lock = mlx5dr_bwc_get_queue_lock(ctx, i);
+		rte_spinlock_lock(queue_lock);
+	}
+}
+
+static void mlx5dr_bwc_unlock_all_queues(struct mlx5dr_context *ctx)
+{
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	rte_spinlock_t *queue_lock;
+	int i;
+
+	for (i = 0; i < bwc_queues; i++) {
+		queue_lock = mlx5dr_bwc_get_queue_lock(ctx, i);
+		rte_spinlock_unlock(queue_lock);
+	}
+}
+
+static void mlx5dr_bwc_matcher_init_attr(struct mlx5dr_matcher_attr *attr,
+					 uint32_t priority,
+					 uint8_t size_log,
+					 bool is_root)
+{
+	memset(attr, 0, sizeof(*attr));
+
+	attr->priority = priority;
+	attr->optimize_using_rule_idx = 0;
+	attr->mode = MLX5DR_MATCHER_RESOURCE_MODE_RULE;
+	attr->optimize_flow_src = MLX5DR_MATCHER_FLOW_SRC_ANY;
+	attr->insert_mode = MLX5DR_MATCHER_INSERT_BY_HASH;
+	attr->distribute_mode = MLX5DR_MATCHER_DISTRIBUTE_BY_HASH;
+	attr->rule.num_log = size_log;
+
+	if (!is_root) {
+		attr->resizable = true;
+		attr->max_num_of_at_attach = MLX5DR_BWC_MATCHER_ATTACH_AT_NUM;
+	}
+}
+
+struct mlx5dr_bwc_matcher *
+mlx5dr_bwc_matcher_create(struct mlx5dr_table *table,
+			  uint32_t priority,
+			  const struct rte_flow_item flow_items[])
+{
+	enum mlx5dr_action_type init_action_types[1] = { MLX5DR_ACTION_TYP_LAST };
+	uint16_t bwc_queues = mlx5dr_bwc_queues(table->ctx);
+	struct mlx5dr_bwc_matcher *bwc_matcher;
+	struct mlx5dr_matcher_attr attr = {0};
+	int i;
+
+	if (!mlx5dr_context_bwc_supported(table->ctx)) {
+		rte_errno = EINVAL;
+		DR_LOG(ERR, "BWC rule: Context created w/o BWC API compatibility");
+		return NULL;
+	}
+
+	bwc_matcher = simple_calloc(1, sizeof(*bwc_matcher));
+	if (!bwc_matcher) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	bwc_matcher->rules = simple_calloc(bwc_queues, sizeof(*bwc_matcher->rules));
+	if (!bwc_matcher->rules) {
+		rte_errno = ENOMEM;
+		goto free_bwc_matcher;
+	}
+
+	for (i = 0; i < bwc_queues; i++)
+		LIST_INIT(&bwc_matcher->rules[i]);
+
+	mlx5dr_bwc_matcher_init_attr(&attr,
+				     priority,
+				     MLX5DR_BWC_MATCHER_INIT_SIZE_LOG,
+				     mlx5dr_table_is_root(table));
+
+	bwc_matcher->mt = mlx5dr_match_template_create(flow_items,
+						       MLX5DR_MATCH_TEMPLATE_FLAG_NONE);
+	if (!bwc_matcher->mt) {
+		rte_errno = EINVAL;
+		goto free_bwc_matcher_rules;
+	}
+
+	bwc_matcher->priority = priority;
+	bwc_matcher->size_log = MLX5DR_BWC_MATCHER_INIT_SIZE_LOG;
+
+	/* create dummy action template */
+	bwc_matcher->at[0] = mlx5dr_action_template_create(init_action_types, 0);
+	bwc_matcher->num_of_at = 1;
+
+	bwc_matcher->matcher = mlx5dr_matcher_create(table,
+						     &bwc_matcher->mt, 1,
+						     &bwc_matcher->at[0],
+						     bwc_matcher->num_of_at,
+						     &attr);
+	if (!bwc_matcher->matcher) {
+		rte_errno = EINVAL;
+		goto free_at;
+	}
+
+	return bwc_matcher;
+
+free_at:
+	mlx5dr_action_template_destroy(bwc_matcher->at[0]);
+	mlx5dr_match_template_destroy(bwc_matcher->mt);
+free_bwc_matcher_rules:
+	simple_free(bwc_matcher->rules);
+free_bwc_matcher:
+	simple_free(bwc_matcher);
+
+	return NULL;
+}
+
+int mlx5dr_bwc_matcher_destroy(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	int i;
+
+	if (bwc_matcher->num_of_rules)
+		DR_LOG(ERR, "BWC matcher destroy: matcher still has %d rules",
+		       bwc_matcher->num_of_rules);
+
+	mlx5dr_matcher_destroy(bwc_matcher->matcher);
+	bwc_matcher->matcher = NULL;
+
+	for (i = 0; i < bwc_matcher->num_of_at; i++)
+		mlx5dr_action_template_destroy(bwc_matcher->at[i]);
+
+	mlx5dr_match_template_destroy(bwc_matcher->mt);
+	simple_free(bwc_matcher->rules);
+	simple_free(bwc_matcher);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_queue_poll(struct mlx5dr_context *ctx,
+		      uint16_t queue_id,
+		      uint32_t *pending_rules,
+		      bool drain)
+{
+	bool queue_full = *pending_rules == MLX5DR_BWC_MATCHER_REHASH_QUEUE_SZ;
+	bool got_comp = *pending_rules >= MLX5DR_BWC_MATCHER_REHASH_BURST_TH;
+	struct rte_flow_op_result comp[MLX5DR_BWC_MATCHER_REHASH_BURST_TH];
+	int ret;
+	int i;
+
+	/* Check if there are any completions at all */
+	if (!got_comp && !drain)
+		return 0;
+
+	while (queue_full || ((got_comp || drain) && *pending_rules)) {
+		ret = mlx5dr_send_queue_poll(ctx, queue_id, comp,
+					     MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+		if (unlikely(ret < 0)) {
+			DR_LOG(ERR, "Rehash error: polling queue %d returned %d\n",
+			       queue_id, ret);
+			return -EINVAL;
+		}
+
+		if (ret) {
+			(*pending_rules) -= ret;
+			for (i = 0; i < ret; i++) {
+				if (unlikely(comp[i].status != RTE_FLOW_OP_SUCCESS))
+					DR_LOG(ERR,
+					       "Rehash error: polling queue %d returned completion with error\n",
+					       queue_id);
+			}
+			queue_full = false;
+		}
+
+		got_comp = !!ret;
+	}
+
+	return 0;
+}
+
+static void
+mlx5dr_bwc_rule_fill_attr(struct mlx5dr_bwc_matcher *bwc_matcher,
+			  uint16_t bwc_queue_idx,
+			  struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+
+	/* no use of INSERT_BY_INDEX in bwc rule */
+	rule_attr->rule_idx = 0;
+
+	/* notify HW at each rule insertion/deletion */
+	rule_attr->burst = 0;
+
+	/* We don't need user data, but the API requires it to exist */
+	rule_attr->user_data = (void *)0xFACADE;
+
+	rule_attr->queue_id = mlx5dr_bwc_get_queue_id(ctx, bwc_queue_idx);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_alloc(void)
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+
+	bwc_rule = simple_calloc(1, sizeof(*bwc_rule));
+	if (unlikely(!bwc_rule))
+		goto out_err;
+
+	bwc_rule->rule = simple_calloc(1, sizeof(*bwc_rule->rule));
+	if (unlikely(!bwc_rule->rule))
+		goto free_rule;
+
+	return bwc_rule;
+
+free_rule:
+	simple_free(bwc_rule);
+out_err:
+	rte_errno = ENOMEM;
+	return NULL;
+}
+
+static void
+mlx5dr_bwc_rule_free(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	if (likely(bwc_rule->rule))
+		simple_free(bwc_rule->rule);
+	simple_free(bwc_rule);
+}
+
+static void
+mlx5dr_bwc_rule_list_add(struct mlx5dr_bwc_rule *bwc_rule, uint16_t idx)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+
+	rte_atomic_fetch_add_explicit(&bwc_matcher->num_of_rules, 1, rte_memory_order_relaxed);
+	bwc_rule->bwc_queue_idx = idx;
+	LIST_INSERT_HEAD(&bwc_matcher->rules[idx], bwc_rule, next);
+}
+
+static void mlx5dr_bwc_rule_list_remove(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+
+	rte_atomic_fetch_sub_explicit(&bwc_matcher->num_of_rules, 1, rte_memory_order_relaxed);
+	LIST_REMOVE(bwc_rule, next);
+}
+
+static int
+mlx5dr_bwc_rule_destroy_hws_async(struct mlx5dr_bwc_rule *bwc_rule,
+				  struct mlx5dr_rule_attr *attr)
+{
+	return mlx5dr_rule_destroy(bwc_rule->rule, attr);
+}
+
+static int
+mlx5dr_bwc_rule_destroy_hws_sync(struct mlx5dr_bwc_rule *bwc_rule,
+				 struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx;
+	struct rte_flow_op_result completion;
+	int ret;
+
+	ret = mlx5dr_bwc_rule_destroy_hws_async(bwc_rule, rule_attr);
+	if (unlikely(ret))
+		return ret;
+
+	do {
+		ret = mlx5dr_send_queue_poll(ctx, rule_attr->queue_id, &completion, 1);
+	} while (ret != 1);
+
+	if (unlikely(completion.status != RTE_FLOW_OP_SUCCESS ||
+		     (bwc_rule->rule->status != MLX5DR_RULE_STATUS_DELETED &&
+		      bwc_rule->rule->status != MLX5DR_RULE_STATUS_DELETING))) {
+		DR_LOG(ERR, "Failed destroying BWC rule: completion %d, rule status %d",
+		       completion.status, bwc_rule->rule->status);
+		rte_errno = EINVAL;
+		return rte_errno;
+	}
+
+	return 0;
+}
+
+static int mlx5dr_bwc_rule_destroy_hws(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	struct mlx5dr_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	uint16_t idx = bwc_rule->bwc_queue_idx;
+	struct mlx5dr_rule_attr attr;
+	rte_spinlock_t *queue_lock;
+	int ret;
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, idx, &attr);
+
+	queue_lock = mlx5dr_bwc_get_queue_lock(ctx, idx);
+
+	rte_spinlock_lock(queue_lock);
+
+	ret = mlx5dr_bwc_rule_destroy_hws_sync(bwc_rule, &attr);
+	mlx5dr_bwc_rule_list_remove(bwc_rule);
+
+	rte_spinlock_unlock(queue_lock);
+
+	mlx5dr_bwc_rule_free(bwc_rule);
+
+	return ret;
+}
+
+static int mlx5dr_bwc_rule_destroy_root(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	int ret;
+
+	ret = mlx5dr_rule_destroy_root_no_comp(bwc_rule->rule);
+
+	mlx5dr_bwc_rule_free(bwc_rule);
+
+	return ret;
+}
+
+int mlx5dr_bwc_rule_destroy(struct mlx5dr_bwc_rule *bwc_rule)
+{
+	if (unlikely(mlx5dr_table_is_root(bwc_rule->bwc_matcher->matcher->tbl)))
+		return mlx5dr_bwc_rule_destroy_root(bwc_rule);
+
+	return mlx5dr_bwc_rule_destroy_hws(bwc_rule);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws_async(struct mlx5dr_bwc_matcher *bwc_matcher,
+				 const struct rte_flow_item flow_items[],
+				 uint8_t at_idx,
+				 struct mlx5dr_rule_action rule_actions[],
+				 struct mlx5dr_rule_attr *rule_attr)
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_alloc();
+	if (unlikely(!bwc_rule))
+		return NULL;
+
+	bwc_rule->bwc_matcher = bwc_matcher;
+
+	ret = mlx5dr_rule_create(bwc_matcher->matcher,
+				 0, /* only one match template supported */
+				 flow_items,
+				 at_idx,
+				 rule_actions,
+				 rule_attr,
+				 bwc_rule->rule);
+
+	if (unlikely(ret)) {
+		mlx5dr_bwc_rule_free(bwc_rule);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_root_sync(struct mlx5dr_bwc_matcher *bwc_matcher,
+				 const struct rte_flow_item flow_items[],
+				 uint8_t num_actions,
+				 struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_alloc();
+	if (unlikely(!bwc_rule)) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	bwc_rule->bwc_matcher = bwc_matcher;
+	bwc_rule->rule->matcher = bwc_matcher->matcher;
+
+	ret = mlx5dr_rule_create_root_no_comp(bwc_rule->rule,
+					      flow_items,
+					      num_actions,
+					      rule_actions);
+	if (unlikely(ret)) {
+		mlx5dr_bwc_rule_free(bwc_rule);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws_sync(struct mlx5dr_bwc_matcher *bwc_matcher,
+				const struct rte_flow_item flow_items[],
+				uint8_t at_idx,
+				struct mlx5dr_rule_action rule_actions[],
+				struct mlx5dr_rule_attr *rule_attr)
+
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	struct rte_flow_op_result completion;
+	struct mlx5dr_bwc_rule *bwc_rule;
+	int ret;
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_async(bwc_matcher, flow_items,
+						    at_idx, rule_actions,
+						    rule_attr);
+	if (unlikely(!bwc_rule))
+		return NULL;
+
+	do {
+		ret = mlx5dr_send_queue_poll(ctx, rule_attr->queue_id, &completion, 1);
+	} while (ret != 1);
+
+	if (unlikely(completion.status != RTE_FLOW_OP_SUCCESS ||
+		     (bwc_rule->rule->status != MLX5DR_RULE_STATUS_CREATED &&
+		      bwc_rule->rule->status != MLX5DR_RULE_STATUS_CREATING))) {
+		DR_LOG(ERR, "Failed creating BWC rule: completion %d, rule status %d",
+		       completion.status, bwc_rule->rule->status);
+		mlx5dr_bwc_rule_free(bwc_rule);
+		return NULL;
+	}
+
+	return bwc_rule;
+}
+
+static bool
+mlx5dr_bwc_matcher_size_maxed_out(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps;
+
+	return bwc_matcher->size_log + MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH >=
+	       caps->ste_alloc_log_max - 1;
+}
+
+static bool
+mlx5dr_bwc_matcher_rehash_size_needed(struct mlx5dr_bwc_matcher *bwc_matcher,
+				      uint32_t num_of_rules)
+{
+	/* size-based rehash for root table is kernel's responsibility */
+	if (unlikely(mlx5dr_table_is_root(bwc_matcher->matcher->tbl)))
+		return false;
+
+	if (unlikely(mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher)))
+		return false;
+
+	if (unlikely((num_of_rules * 100 / MLX5DR_BWC_MATCHER_REHASH_PERCENT_TH) >=
+		     (1UL << bwc_matcher->size_log)))
+		return true;
+
+	return false;
+}
+
+static void
+mlx5dr_bwc_rule_actions_to_action_types(struct mlx5dr_rule_action rule_actions[],
+					enum mlx5dr_action_type action_types[])
+{
+	int i = 0;
+
+	for (i = 0;
+	     rule_actions[i].action && (rule_actions[i].action->type != MLX5DR_ACTION_TYP_LAST);
+	     i++) {
+		action_types[i] = (enum mlx5dr_action_type)rule_actions[i].action->type;
+	}
+
+	action_types[i] = MLX5DR_ACTION_TYP_LAST;
+}
+
+static int
+mlx5dr_bwc_rule_actions_num(struct mlx5dr_rule_action rule_actions[])
+{
+	int i = 0;
+
+	while (rule_actions[i].action &&
+	       (rule_actions[i].action->type != MLX5DR_ACTION_TYP_LAST))
+		i++;
+
+	return i;
+}
+
+static int
+mlx5dr_bwc_matcher_extend_at(struct mlx5dr_bwc_matcher *bwc_matcher,
+			     struct mlx5dr_rule_action rule_actions[])
+{
+	enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS];
+
+	mlx5dr_bwc_rule_actions_to_action_types(rule_actions, action_types);
+
+	bwc_matcher->at[bwc_matcher->num_of_at] =
+		mlx5dr_action_template_create(action_types, 0);
+
+	if (unlikely(!bwc_matcher->at[bwc_matcher->num_of_at])) {
+		rte_errno = ENOMEM;
+		return rte_errno;
+	}
+
+	bwc_matcher->num_of_at++;
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_extend_size(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps;
+
+	if (unlikely(mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher))) {
+		DR_LOG(ERR, "Can't resize matcher: depth exceeds limit %d",
+		       caps->rtc_log_depth_max);
+		return -ENOMEM;
+	}
+
+	bwc_matcher->size_log =
+		RTE_MIN(bwc_matcher->size_log + MLX5DR_BWC_MATCHER_SIZE_LOG_STEP,
+			caps->ste_alloc_log_max - MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_find_at(struct mlx5dr_bwc_matcher *bwc_matcher,
+			   struct mlx5dr_rule_action rule_actions[])
+{
+	enum mlx5dr_action_type *action_type_arr;
+	int i, j;
+
+	/* start from index 1 - first action template is a dummy */
+	for (i = 1; i < bwc_matcher->num_of_at; i++) {
+		j = 0;
+		action_type_arr = bwc_matcher->at[i]->action_type_arr;
+
+		while (rule_actions[j].action &&
+		       rule_actions[j].action->type != MLX5DR_ACTION_TYP_LAST) {
+			if (action_type_arr[j] != rule_actions[j].action->type)
+				break;
+			j++;
+		}
+
+		if (action_type_arr[j] == MLX5DR_ACTION_TYP_LAST &&
+		    (!rule_actions[j].action ||
+		     rule_actions[j].action->type == MLX5DR_ACTION_TYP_LAST))
+			return i;
+	}
+
+	return -1;
+}
+
+static int
+mlx5dr_bwc_matcher_move_all(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	uint16_t bwc_queues = mlx5dr_bwc_queues(ctx);
+	struct mlx5dr_bwc_rule **bwc_rules;
+	struct mlx5dr_rule_attr rule_attr;
+	uint32_t *pending_rules;
+	bool all_done;
+	int i, j, ret;
+
+	if (mlx5dr_table_is_root(bwc_matcher->matcher->tbl)) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, 0, &rule_attr);
+
+	pending_rules = simple_calloc(bwc_queues, sizeof(*pending_rules));
+	if (!pending_rules) {
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+
+	bwc_rules = simple_calloc(bwc_queues, sizeof(*bwc_rules));
+	if (!bwc_rules) {
+		rte_errno = ENOMEM;
+		goto free_pending_rules;
+	}
+
+	for (i = 0; i < bwc_queues; i++) {
+		if (LIST_EMPTY(&bwc_matcher->rules[i]))
+			bwc_rules[i] = NULL;
+		else
+			bwc_rules[i] = LIST_FIRST(&bwc_matcher->rules[i]);
+	}
+
+	do {
+		all_done = true;
+
+		for (i = 0; i < bwc_queues; i++) {
+			rule_attr.queue_id = mlx5dr_bwc_get_queue_id(ctx, i);
+
+			for (j = 0; j < MLX5DR_BWC_MATCHER_REHASH_BURST_TH && bwc_rules[i]; j++) {
+				rule_attr.burst = !!((j + 1) % MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+				ret = mlx5dr_matcher_resize_rule_move(bwc_matcher->matcher,
+								      bwc_rules[i]->rule,
+								      &rule_attr);
+				if (ret) {
+					DR_LOG(ERR, "Moving BWC rule failed during rehash - %d",
+					       ret);
+					rte_errno = ENOMEM;
+					goto free_bwc_rules;
+				}
+
+				all_done = false;
+				pending_rules[i]++;
+				bwc_rules[i] = LIST_NEXT(bwc_rules[i], next);
+
+				mlx5dr_bwc_queue_poll(ctx, rule_attr.queue_id,
+						      &pending_rules[i], false);
+			}
+		}
+	} while (!all_done);
+
+	/* drain all the bwc queues */
+	for (i = 0; i < bwc_queues; i++) {
+		if (pending_rules[i]) {
+			uint16_t queue_id = mlx5dr_bwc_get_queue_id(ctx, i);
+			mlx5dr_send_engine_flush_queue(&ctx->send_queue[queue_id]);
+			mlx5dr_bwc_queue_poll(ctx, queue_id,
+					      &pending_rules[i], true);
+		}
+	}
+
+	rte_errno = 0;
+
+free_bwc_rules:
+	simple_free(bwc_rules);
+free_pending_rules:
+	simple_free(pending_rules);
+
+	return -rte_errno;
+}
+
+static int
+mlx5dr_bwc_matcher_move(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	struct mlx5dr_matcher_attr matcher_attr = {0};
+	struct mlx5dr_matcher *old_matcher;
+	struct mlx5dr_matcher *new_matcher;
+	int ret;
+
+	mlx5dr_bwc_matcher_init_attr(&matcher_attr,
+				     bwc_matcher->priority,
+				     bwc_matcher->size_log,
+				     mlx5dr_table_is_root(bwc_matcher->matcher->tbl));
+
+	old_matcher = bwc_matcher->matcher;
+	new_matcher = mlx5dr_matcher_create(old_matcher->tbl,
+					    &bwc_matcher->mt, 1,
+					    bwc_matcher->at,
+					    bwc_matcher->num_of_at,
+					    &matcher_attr);
+	if (!new_matcher) {
+		DR_LOG(ERR, "Rehash error: matcher creation failed");
+		return -ENOMEM;
+	}
+
+	ret = mlx5dr_matcher_resize_set_target(old_matcher, new_matcher);
+	if (ret) {
+		DR_LOG(ERR, "Rehash error: failed setting resize target");
+		return ret;
+	}
+
+	ret = mlx5dr_bwc_matcher_move_all(bwc_matcher);
+	if (ret) {
+		DR_LOG(ERR, "Rehash error: moving rules failed");
+		return -ENOMEM;
+	}
+
+	bwc_matcher->matcher = new_matcher;
+	mlx5dr_matcher_destroy(old_matcher);
+
+	return 0;
+}
+
+static int
+mlx5dr_bwc_matcher_rehash(struct mlx5dr_bwc_matcher *bwc_matcher, bool rehash_size)
+{
+	uint32_t num_of_rules;
+	int ret;
+
+	/* If the current matcher size is already at its max size, we can't
+	 * do the rehash. Skip it and try adding the rule again - perhaps
+	 * there was some change.
+	 */
+	if (mlx5dr_bwc_matcher_size_maxed_out(bwc_matcher))
+		return 0;
+
+	/* It is possible that other rule has already performed rehash.
+	 * Need to check again if we really need rehash.
+	 * If the reason for rehash was size, but not any more - skip rehash.
+	 */
+	num_of_rules = rte_atomic_load_explicit(&bwc_matcher->num_of_rules,
+						rte_memory_order_relaxed);
+	if (rehash_size &&
+	    !mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules))
+		return 0;
+
+	/* Now we're done all the checking - do the rehash:
+	 *  - extend match RTC size
+	 *  - create new matcher
+	 *  - move all the rules to the new matcher
+	 *  - destroy the old matcher
+	 */
+
+	ret = mlx5dr_bwc_matcher_extend_size(bwc_matcher);
+	if (ret)
+		return ret;
+
+	return mlx5dr_bwc_matcher_move(bwc_matcher);
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_root(struct mlx5dr_bwc_matcher *bwc_matcher,
+			    const struct rte_flow_item flow_items[],
+			    struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_bwc_rule *bwc_rule;
+
+	bwc_rule = mlx5dr_bwc_rule_create_root_sync(bwc_matcher,
+						    flow_items,
+						    mlx5dr_bwc_rule_actions_num(rule_actions),
+						    rule_actions);
+
+	if (unlikely(!bwc_rule))
+		DR_LOG(ERR, "BWC rule: failed creating rule on root tbl");
+
+	return bwc_rule;
+}
+
+static struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create_hws(struct mlx5dr_bwc_matcher *bwc_matcher,
+			   const struct rte_flow_item flow_items[],
+			   struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+	struct mlx5dr_bwc_rule *bwc_rule = NULL;
+	struct mlx5dr_rule_attr rule_attr;
+	rte_spinlock_t *queue_lock;
+	bool rehash_size = false;
+	uint32_t num_of_rules;
+	uint16_t idx;
+	int at_idx;
+	int ret;
+
+	idx = mlx5dr_bwc_gen_queue_idx(ctx);
+
+	mlx5dr_bwc_rule_fill_attr(bwc_matcher, idx, &rule_attr);
+
+	queue_lock = mlx5dr_bwc_get_queue_lock(ctx, idx);
+
+	rte_spinlock_lock(queue_lock);
+
+	/* check if rehash needed due to missing action template */
+	at_idx = mlx5dr_bwc_matcher_find_at(bwc_matcher, rule_actions);
+	if (at_idx < 0) {
+		/* we need to extend BWC matcher action templates array */
+		rte_spinlock_unlock(queue_lock);
+		mlx5dr_bwc_lock_all_queues(ctx);
+
+		ret = mlx5dr_bwc_matcher_extend_at(bwc_matcher, rule_actions);
+		if (unlikely(ret)) {
+			mlx5dr_bwc_unlock_all_queues(ctx);
+			rte_errno = EINVAL;
+			DR_LOG(ERR, "BWC rule: failed extending action template - %d", ret);
+			return NULL;
+		}
+
+		/* action templates array was extended, we need the last idx */
+		at_idx = bwc_matcher->num_of_at - 1;
+
+		ret = mlx5dr_matcher_attach_at(bwc_matcher->matcher,
+					       bwc_matcher->at[at_idx]);
+		if (unlikely(ret)) {
+			mlx5dr_action_template_destroy(bwc_matcher->at[at_idx]);
+			bwc_matcher->at[at_idx] = NULL;
+			bwc_matcher->num_of_at--;
+
+			mlx5dr_bwc_unlock_all_queues(ctx);
+			DR_LOG(ERR, "BWC rule: failed attaching action template - %d", ret);
+			return NULL;
+		}
+
+		mlx5dr_bwc_unlock_all_queues(ctx);
+		rte_spinlock_lock(queue_lock);
+	}
+
+	/* check if number of rules require rehash */
+	num_of_rules = rte_atomic_load_explicit(&bwc_matcher->num_of_rules,
+						rte_memory_order_relaxed);
+	if (mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules)) {
+		rehash_size = true;
+		goto rehash;
+	}
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_sync(bwc_matcher,
+						   flow_items,
+						   at_idx,
+						   rule_actions,
+						   &rule_attr);
+
+	if (likely(bwc_rule)) {
+		mlx5dr_bwc_rule_list_add(bwc_rule, idx);
+		rte_spinlock_unlock(queue_lock);
+		return bwc_rule; /* rule inserted successfully */
+	}
+
+	/* At this point the rule wasn't added.
+	 * It could be because there was collision, or some other problem.
+	 * If we don't dive deeper than API, the only thing we know is that
+	 * the status of completion is RTE_FLOW_OP_ERROR.
+	 * Try rehash and insert rule again - last chance.
+	 */
+
+rehash:
+	rte_spinlock_unlock(queue_lock);
+
+	mlx5dr_bwc_lock_all_queues(ctx);
+	ret = mlx5dr_bwc_matcher_rehash(bwc_matcher, rehash_size);
+	mlx5dr_bwc_unlock_all_queues(ctx);
+
+	if (ret) {
+		DR_LOG(ERR, "BWC rule insertion: rehash failed - %d", ret);
+		return NULL;
+	}
+
+	/* Rehash done, but we still have that pesky rule to add */
+	rte_spinlock_lock(queue_lock);
+
+	bwc_rule = mlx5dr_bwc_rule_create_hws_sync(bwc_matcher,
+						   flow_items,
+						   at_idx,
+						   rule_actions,
+						   &rule_attr);
+
+	if (unlikely(!bwc_rule)) {
+		rte_spinlock_unlock(queue_lock);
+		DR_LOG(ERR, "BWC rule insertion failed");
+		return NULL;
+	}
+
+	mlx5dr_bwc_rule_list_add(bwc_rule, idx);
+	rte_spinlock_unlock(queue_lock);
+
+	return bwc_rule;
+}
+
+struct mlx5dr_bwc_rule *
+mlx5dr_bwc_rule_create(struct mlx5dr_bwc_matcher *bwc_matcher,
+		       const struct rte_flow_item flow_items[],
+		       struct mlx5dr_rule_action rule_actions[])
+{
+	struct mlx5dr_context *ctx = bwc_matcher->matcher->tbl->ctx;
+
+	if (unlikely(!mlx5dr_context_bwc_supported(ctx))) {
+		rte_errno = EINVAL;
+		DR_LOG(ERR, "BWC rule: Context created w/o BWC API compatibility");
+		return NULL;
+	}
+
+	if (unlikely(mlx5dr_table_is_root(bwc_matcher->matcher->tbl)))
+		return mlx5dr_bwc_rule_create_root(bwc_matcher,
+						   flow_items,
+						   rule_actions);
+
+	return mlx5dr_bwc_rule_create_hws(bwc_matcher,
+					  flow_items,
+					  rule_actions);
+}
diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.h b/drivers/net/mlx5/hws/mlx5dr_bwc.h
new file mode 100644
index 0000000000..648443861b
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2023 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5DR_BWC_H_
+#define MLX5DR_BWC_H_
+
+#define MLX5DR_BWC_MATCHER_INIT_SIZE_LOG 1
+#define MLX5DR_BWC_MATCHER_SIZE_LOG_STEP 1
+#define MLX5DR_BWC_MATCHER_REHASH_PERCENT_TH 70
+#define MLX5DR_BWC_MATCHER_REHASH_BURST_TH 32
+#define MLX5DR_BWC_MATCHER_REHASH_QUEUE_SZ 256
+#define MLX5DR_BWC_MATCHER_ATTACH_AT_NUM 255
+
+struct mlx5dr_bwc_matcher {
+	struct mlx5dr_matcher *matcher;
+	struct mlx5dr_match_template *mt;
+	struct mlx5dr_action_template *at[MLX5DR_BWC_MATCHER_ATTACH_AT_NUM];
+	uint8_t num_of_at;
+	uint32_t priority;
+	uint8_t size_log;
+	RTE_ATOMIC(uint32_t)num_of_rules; /* atomically accessed */
+	LIST_HEAD(rule_head, mlx5dr_bwc_rule) * rules;
+};
+
+struct mlx5dr_bwc_rule {
+	struct mlx5dr_bwc_matcher *bwc_matcher;
+	struct mlx5dr_rule *rule;
+	struct rte_flow_item *flow_items;
+	uint16_t bwc_queue_idx;
+	LIST_ENTRY(mlx5dr_bwc_rule) next;
+};
+
+#endif /* MLX5DR_BWC_H_ */
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.c b/drivers/net/mlx5/hws/mlx5dr_context.c
index 7f120b3b1b..db5e72927a 100644
--- a/drivers/net/mlx5/hws/mlx5dr_context.c
+++ b/drivers/net/mlx5/hws/mlx5dr_context.c
@@ -176,6 +176,9 @@ static int mlx5dr_context_init_hws(struct mlx5dr_context *ctx,
 	if (ret)
 		goto uninit_pd;
 
+	if (attr->bwc)
+		ctx->flags |= MLX5DR_CONTEXT_FLAG_BWC_SUPPORT;
+
 	ret = mlx5dr_send_queues_open(ctx, attr->queues, attr->queue_size);
 	if (ret)
 		goto pools_uninit;
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.h b/drivers/net/mlx5/hws/mlx5dr_context.h
index f476c2308c..7678232ecc 100644
--- a/drivers/net/mlx5/hws/mlx5dr_context.h
+++ b/drivers/net/mlx5/hws/mlx5dr_context.h
@@ -8,6 +8,7 @@
 enum mlx5dr_context_flags {
 	MLX5DR_CONTEXT_FLAG_HWS_SUPPORT = 1 << 0,
 	MLX5DR_CONTEXT_FLAG_PRIVATE_PD = 1 << 1,
+	MLX5DR_CONTEXT_FLAG_BWC_SUPPORT = 1 << 2,
 };
 
 enum mlx5dr_context_shared_stc_type {
@@ -44,6 +45,7 @@ struct mlx5dr_context {
 	enum mlx5dr_context_flags flags;
 	struct mlx5dr_send_engine *send_queue;
 	size_t queues;
+	rte_spinlock_t *bwc_send_queue_locks;
 	LIST_HEAD(table_head, mlx5dr_table) head;
 };
 
@@ -52,6 +54,11 @@ static inline bool mlx5dr_context_shared_gvmi_used(struct mlx5dr_context *ctx)
 	return ctx->local_ibv_ctx ? true : false;
 }
 
+static inline bool mlx5dr_context_bwc_supported(struct mlx5dr_context *ctx)
+{
+	return ctx->flags & MLX5DR_CONTEXT_FLAG_BWC_SUPPORT;
+}
+
 static inline struct ibv_context *
 mlx5dr_context_get_local_ibv(struct mlx5dr_context *ctx)
 {
diff --git a/drivers/net/mlx5/hws/mlx5dr_internal.h b/drivers/net/mlx5/hws/mlx5dr_internal.h
index b9efdc4a9a..2abc516b5e 100644
--- a/drivers/net/mlx5/hws/mlx5dr_internal.h
+++ b/drivers/net/mlx5/hws/mlx5dr_internal.h
@@ -39,6 +39,7 @@
 #include "mlx5dr_debug.h"
 #include "mlx5dr_pat_arg.h"
 #include "mlx5dr_crc32.h"
+#include "mlx5dr_bwc.h"
 
 #define W_SIZE		2
 #define DW_SIZE		4
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c b/drivers/net/mlx5/hws/mlx5dr_send.c
index 2942668e76..0120f03a48 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.c
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -971,10 +971,42 @@ static void __mlx5dr_send_queues_close(struct mlx5dr_context *ctx, uint16_t queu
 		mlx5dr_send_queue_close(&ctx->send_queue[queues]);
 }
 
+static int mlx5dr_bwc_send_queues_init(struct mlx5dr_context *ctx)
+{
+	int bwc_queues = ctx->queues - 1;
+	int i;
+
+	if (!mlx5dr_context_bwc_supported(ctx))
+		return 0;
+
+	ctx->queues += bwc_queues;
+
+	ctx->bwc_send_queue_locks = simple_calloc(bwc_queues,
+						  sizeof(*ctx->bwc_send_queue_locks));
+	if (!ctx->bwc_send_queue_locks) {
+		rte_errno = ENOMEM;
+		return rte_errno;
+	}
+
+	for (i = 0; i < bwc_queues; i++)
+		rte_spinlock_init(&ctx->bwc_send_queue_locks[i]);
+
+	return 0;
+}
+
+static void mlx5dr_send_queues_bwc_locks_destroy(struct mlx5dr_context *ctx)
+{
+	if (!mlx5dr_context_bwc_supported(ctx))
+		return;
+
+	simple_free(ctx->bwc_send_queue_locks);
+}
+
 void mlx5dr_send_queues_close(struct mlx5dr_context *ctx)
 {
 	__mlx5dr_send_queues_close(ctx, ctx->queues);
 	simple_free(ctx->send_queue);
+	mlx5dr_send_queues_bwc_locks_destroy(ctx);
 }
 
 int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
@@ -987,10 +1019,16 @@ int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
 	/* Open one extra queue for control path */
 	ctx->queues = queues + 1;
 
+	/* open a separate set of queues and locks for bwc API */
+	err = mlx5dr_bwc_send_queues_init(ctx);
+	if (err)
+		return err;
+
 	ctx->send_queue = simple_calloc(ctx->queues, sizeof(*ctx->send_queue));
 	if (!ctx->send_queue) {
 		rte_errno = ENOMEM;
-		return rte_errno;
+		err = rte_errno;
+		goto free_bwc_locks;
 	}
 
 	for (i = 0; i < ctx->queues; i++) {
@@ -1006,6 +1044,9 @@ int mlx5dr_send_queues_open(struct mlx5dr_context *ctx,
 
 	simple_free(ctx->send_queue);
 
+free_bwc_locks:
+	mlx5dr_send_queues_bwc_locks_destroy(ctx);
+
 	return err;
 }
 
-- 
2.21.0

Re: [PATCH v4 0/3] HWS non-template support

[Raslan Darawsheh]

Hi,

From: Maayan Kashani <mkashani@nvidia.com>
Sent: Thursday, June 6, 2024 1:31 PM
To: dev@dpdk.org
Cc: Maayan Kashani; Dariusz Sosnowski; Raslan Darawsheh
Subject: [PATCH v4 0/3] HWS non-template support

Initial coding to support non-template in mlxdr layer.

Yevgeny Kliteynik (3):
  net/mlx5/hws: split the root rule creation and destruction
  net/mlx5/hws: add support for action type LAST
  net/mlx5/hws: add support for backward-compatible API

 drivers/net/mlx5/hws/meson.build       |   1 +
 drivers/net/mlx5/hws/mlx5dr.h          |  87 +++
 drivers/net/mlx5/hws/mlx5dr_action.c   |   9 +
 drivers/net/mlx5/hws/mlx5dr_bwc.c      | 898 +++++++++++++++++++++++++
 drivers/net/mlx5/hws/mlx5dr_bwc.h      |  34 +
 drivers/net/mlx5/hws/mlx5dr_context.c  |   3 +
 drivers/net/mlx5/hws/mlx5dr_context.h  |   7 +
 drivers/net/mlx5/hws/mlx5dr_internal.h |   1 +
 drivers/net/mlx5/hws/mlx5dr_rule.c     |  49 +-
 drivers/net/mlx5/hws/mlx5dr_rule.h     |   7 +
 drivers/net/mlx5/hws/mlx5dr_send.c     |  43 +-
 11 files changed, 1124 insertions(+), 15 deletions(-)
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.c
 create mode 100644 drivers/net/mlx5/hws/mlx5dr_bwc.h

--
2.21.0


series applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh