[PATCH 1/7] net/mlx5/hws: fix bug in matcher disconnect error flow

[PATCH 1/7] net/mlx5/hws: fix bug in matcher disconnect error flow

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

If error happens during disconnect of the first matcher in the
list, the matcher should be reconnected back as the first matcher.

Fixes: b81f95ca770d ("net/mlx5/hws: support default miss table")
Cc: stable@dpdk.org
Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_matcher.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index 79e7401dfd..2a84145566 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -340,7 +340,7 @@ static int mlx5dr_matcher_disconnect(struct mlx5dr_matcher *matcher)
 	return 0;
 
 matcher_reconnect:
-	if (LIST_EMPTY(&tbl->head))
+	if (LIST_EMPTY(&tbl->head) || prev_matcher == matcher)
 		LIST_INSERT_HEAD(&matcher->tbl->head, matcher, next);
 	else
 		LIST_INSERT_AFTER(prev_matcher, matcher, next);
-- 
2.25.1


From 83beeaad51e7175b846b2d61b99787ef75b340f7 Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn@nvidia.com>
Date: Tue, 5 Mar 2024 02:30:46 +0200
Subject: [PATCH 2/7] net/mlx5/hws: bwc - make burst threshold dynamic

BWC rules rehash process has burst threshold, where the rule
requires completion. This threshold was constant number, and
it can cause rehash fail if this threshold is actually bigger
than the queue size - the queue would end up full.
This patch fixes the threshold to be dynamic and to take into
consideration queue size.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_bwc.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.c b/drivers/net/mlx5/hws/mlx5dr_bwc.c
index eef3053ee0..bfc7dbf100 100644
--- a/drivers/net/mlx5/hws/mlx5dr_bwc.c
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.c
@@ -22,6 +22,13 @@ mlx5dr_bwc_get_queue_id(struct mlx5dr_context *ctx, uint16_t idx)
 	return idx + mlx5dr_bwc_queues(ctx);
 }
 
+static uint16_t
+mlx5dr_bwc_get_burst_th(struct mlx5dr_context *ctx, uint16_t queue_id)
+{
+	return RTE_MIN(ctx->send_queue[queue_id].num_entries / 2,
+		       MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+}
+
 static rte_spinlock_t *
 mlx5dr_bwc_get_queue_lock(struct mlx5dr_context *ctx, uint16_t idx)
 {
@@ -175,8 +182,9 @@ mlx5dr_bwc_queue_poll(struct mlx5dr_context *ctx,
 		      bool drain)
 {
 	bool queue_full = *pending_rules == MLX5DR_BWC_MATCHER_REHASH_QUEUE_SZ;
-	bool got_comp = *pending_rules >= MLX5DR_BWC_MATCHER_REHASH_BURST_TH;
 	struct rte_flow_op_result comp[MLX5DR_BWC_MATCHER_REHASH_BURST_TH];
+	uint16_t burst_th = mlx5dr_bwc_get_burst_th(ctx, queue_id);
+	bool got_comp = *pending_rules >= burst_th;
 	int ret;
 	int i;
 
@@ -185,8 +193,7 @@ mlx5dr_bwc_queue_poll(struct mlx5dr_context *ctx,
 		return 0;
 
 	while (queue_full || ((got_comp || drain) && *pending_rules)) {
-		ret = mlx5dr_send_queue_poll(ctx, queue_id, comp,
-					     MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+		ret = mlx5dr_send_queue_poll(ctx, queue_id, comp, burst_th);
 		if (unlikely(ret < 0)) {
 			DR_LOG(ERR, "Rehash error: polling queue %d returned %d\n",
 			       queue_id, ret);
@@ -583,6 +590,7 @@ mlx5dr_bwc_matcher_move_all(struct mlx5dr_bwc_matcher *bwc_matcher)
 	struct mlx5dr_bwc_rule **bwc_rules;
 	struct mlx5dr_rule_attr rule_attr;
 	uint32_t *pending_rules;
+	uint16_t burst_th;
 	bool all_done;
 	int i, j, ret;
 
@@ -617,9 +625,10 @@ mlx5dr_bwc_matcher_move_all(struct mlx5dr_bwc_matcher *bwc_matcher)
 
 		for (i = 0; i < bwc_queues; i++) {
 			rule_attr.queue_id = mlx5dr_bwc_get_queue_id(ctx, i);
+			burst_th = mlx5dr_bwc_get_burst_th(ctx, rule_attr.queue_id);
 
-			for (j = 0; j < MLX5DR_BWC_MATCHER_REHASH_BURST_TH && bwc_rules[i]; j++) {
-				rule_attr.burst = !!((j + 1) % MLX5DR_BWC_MATCHER_REHASH_BURST_TH);
+			for (j = 0; j < burst_th && bwc_rules[i]; j++) {
+				rule_attr.burst = !!((j + 1) % burst_th);
 				ret = mlx5dr_matcher_resize_rule_move(bwc_matcher->matcher,
 								      bwc_rules[i]->rule,
 								      &rule_attr);
-- 
2.25.1


From d056420b64d44928a5596b3965f6e9a89699ace4 Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn@nvidia.com>
Date: Tue, 5 Mar 2024 01:26:15 +0200
Subject: [PATCH 3/7] net/mlx5/hws: bwc - adding rules with larger action
 template

Current BWC implementation doesn't support a case when a rule is added
with AT that requires more action STEs than any other AT that already
exists on this matcher.
This patch adds this support: if such AT is required, we will add the
new AT and do full rehash to a new matcher that is created with all the
action templates, including the new larger AT.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_bwc.c     | 68 ++++++++++++++++++++-------
 drivers/net/mlx5/hws/mlx5dr_matcher.c |  4 +-
 2 files changed, 52 insertions(+), 20 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.c b/drivers/net/mlx5/hws/mlx5dr_bwc.c
index bfc7dbf100..8a8a143f17 100644
--- a/drivers/net/mlx5/hws/mlx5dr_bwc.c
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.c
@@ -712,7 +712,7 @@ mlx5dr_bwc_matcher_move(struct mlx5dr_bwc_matcher *bwc_matcher)
 }
 
 static int
-mlx5dr_bwc_matcher_rehash(struct mlx5dr_bwc_matcher *bwc_matcher, bool rehash_size)
+mlx5dr_bwc_matcher_rehash_size(struct mlx5dr_bwc_matcher *bwc_matcher)
 {
 	uint32_t num_of_rules;
 	int ret;
@@ -730,8 +730,7 @@ mlx5dr_bwc_matcher_rehash(struct mlx5dr_bwc_matcher *bwc_matcher, bool rehash_si
 	 */
 	num_of_rules = rte_atomic_load_explicit(&bwc_matcher->num_of_rules,
 						rte_memory_order_relaxed);
-	if (rehash_size &&
-	    !mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules))
+	if (!mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules))
 		return 0;
 
 	/* Now we're done all the checking - do the rehash:
@@ -748,6 +747,19 @@ mlx5dr_bwc_matcher_rehash(struct mlx5dr_bwc_matcher *bwc_matcher, bool rehash_si
 	return mlx5dr_bwc_matcher_move(bwc_matcher);
 }
 
+static int
+mlx5dr_bwc_matcher_rehash_at(struct mlx5dr_bwc_matcher *bwc_matcher)
+{
+	/* Rehash by action template doesn't require any additional checking.
+	 * The bwc_matcher already contains the new action template.
+	 * Just do the usual rehash:
+	 *  - create new matcher
+	 *  - move all the rules to the new matcher
+	 *  - destroy the old matcher
+	 */
+	return mlx5dr_bwc_matcher_move(bwc_matcher);
+}
+
 static struct mlx5dr_bwc_rule *
 mlx5dr_bwc_rule_create_root(struct mlx5dr_bwc_matcher *bwc_matcher,
 			    const struct rte_flow_item flow_items[],
@@ -775,7 +787,6 @@ mlx5dr_bwc_rule_create_hws(struct mlx5dr_bwc_matcher *bwc_matcher,
 	struct mlx5dr_bwc_rule *bwc_rule = NULL;
 	struct mlx5dr_rule_attr rule_attr;
 	rte_spinlock_t *queue_lock;
-	bool rehash_size = false;
 	uint32_t num_of_rules;
 	uint16_t idx;
 	int at_idx;
@@ -791,7 +802,7 @@ mlx5dr_bwc_rule_create_hws(struct mlx5dr_bwc_matcher *bwc_matcher,
 
 	/* check if rehash needed due to missing action template */
 	at_idx = mlx5dr_bwc_matcher_find_at(bwc_matcher, rule_actions);
-	if (at_idx < 0) {
+	if (unlikely(at_idx < 0)) {
 		/* we need to extend BWC matcher action templates array */
 		rte_spinlock_unlock(queue_lock);
 		mlx5dr_bwc_lock_all_queues(ctx);
@@ -810,13 +821,22 @@ mlx5dr_bwc_rule_create_hws(struct mlx5dr_bwc_matcher *bwc_matcher,
 		ret = mlx5dr_matcher_attach_at(bwc_matcher->matcher,
 					       bwc_matcher->at[at_idx]);
 		if (unlikely(ret)) {
-			mlx5dr_action_template_destroy(bwc_matcher->at[at_idx]);
-			bwc_matcher->at[at_idx] = NULL;
-			bwc_matcher->num_of_at--;
-
-			mlx5dr_bwc_unlock_all_queues(ctx);
-			DR_LOG(ERR, "BWC rule: failed attaching action template - %d", ret);
-			return NULL;
+			/* Action template attach failed, possibly due to
+			 * requiring more action STEs.
+			 * Need to attempt creating new matcher with all
+			 * the action templates, including the new one.
+			 */
+			ret = mlx5dr_bwc_matcher_rehash_at(bwc_matcher);
+			if (unlikely(ret)) {
+				mlx5dr_action_template_destroy(bwc_matcher->at[at_idx]);
+				bwc_matcher->at[at_idx] = NULL;
+				bwc_matcher->num_of_at--;
+
+				mlx5dr_bwc_unlock_all_queues(ctx);
+
+				DR_LOG(ERR, "BWC rule insertion: rehash AT failed - %d", ret);
+				return NULL;
+			}
 		}
 
 		mlx5dr_bwc_unlock_all_queues(ctx);
@@ -826,9 +846,22 @@ mlx5dr_bwc_rule_create_hws(struct mlx5dr_bwc_matcher *bwc_matcher,
 	/* check if number of rules require rehash */
 	num_of_rules = rte_atomic_load_explicit(&bwc_matcher->num_of_rules,
 						rte_memory_order_relaxed);
-	if (mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules)) {
-		rehash_size = true;
-		goto rehash;
+	if (unlikely(mlx5dr_bwc_matcher_rehash_size_needed(bwc_matcher, num_of_rules))) {
+		rte_spinlock_unlock(queue_lock);
+
+		mlx5dr_bwc_lock_all_queues(ctx);
+		ret = mlx5dr_bwc_matcher_rehash_size(bwc_matcher);
+		mlx5dr_bwc_unlock_all_queues(ctx);
+
+		if (ret) {
+			DR_LOG(ERR, "BWC rule insertion: rehash size [%d -> %d] failed - %d",
+			       bwc_matcher->size_log - MLX5DR_BWC_MATCHER_SIZE_LOG_STEP,
+			       bwc_matcher->size_log,
+			       ret);
+			return NULL;
+		}
+
+		rte_spinlock_lock(queue_lock);
 	}
 
 	bwc_rule = mlx5dr_bwc_rule_create_hws_sync(bwc_matcher,
@@ -847,14 +880,13 @@ mlx5dr_bwc_rule_create_hws(struct mlx5dr_bwc_matcher *bwc_matcher,
 	 * It could be because there was collision, or some other problem.
 	 * If we don't dive deeper than API, the only thing we know is that
 	 * the status of completion is RTE_FLOW_OP_ERROR.
-	 * Try rehash and insert rule again - last chance.
+	 * Try rehash by size and insert rule again - last chance.
 	 */
 
-rehash:
 	rte_spinlock_unlock(queue_lock);
 
 	mlx5dr_bwc_lock_all_queues(ctx);
-	ret = mlx5dr_bwc_matcher_rehash(bwc_matcher, rehash_size);
+	ret = mlx5dr_bwc_matcher_rehash_size(bwc_matcher);
 	mlx5dr_bwc_unlock_all_queues(ctx);
 
 	if (ret) {
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index 2a84145566..aa5ab96d23 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -1347,7 +1347,7 @@ int mlx5dr_matcher_attach_at(struct mlx5dr_matcher *matcher,
 	int ret;
 
 	if (!matcher->attr.max_num_of_at_attach) {
-		DR_LOG(ERR, "Num of current at (%d) exceed allowed value",
+		DR_LOG(DEBUG, "Num of current at (%d) exceed allowed value",
 		       matcher->num_of_at);
 		rte_errno = ENOTSUP;
 		return -rte_errno;
@@ -1359,7 +1359,7 @@ int mlx5dr_matcher_attach_at(struct mlx5dr_matcher *matcher,
 
 	required_stes = at->num_of_action_stes - (!is_jumbo || at->only_term);
 	if (matcher->action_ste.max_stes < required_stes) {
-		DR_LOG(ERR, "Required STEs [%d] exceeds initial action template STE [%d]",
+		DR_LOG(DEBUG, "Required STEs [%d] exceeds initial action template STE [%d]",
 		       required_stes, matcher->action_ste.max_stes);
 		rte_errno = ENOMEM;
 		return -rte_errno;
-- 
2.25.1


From ce404ce362daad2dae2cabde9268c174715e123e Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn@nvidia.com>
Date: Wed, 13 Mar 2024 02:01:07 +0200
Subject: [PATCH 4/7] net/mlx5/hws: bwc - abort rehash on completion with error

If during rehash there is a completion with error on moving
rule to a new matcher, abort the rehash and return error.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_bwc.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_bwc.c b/drivers/net/mlx5/hws/mlx5dr_bwc.c
index 8a8a143f17..e0b6390ed9 100644
--- a/drivers/net/mlx5/hws/mlx5dr_bwc.c
+++ b/drivers/net/mlx5/hws/mlx5dr_bwc.c
@@ -185,6 +185,7 @@ mlx5dr_bwc_queue_poll(struct mlx5dr_context *ctx,
 	struct rte_flow_op_result comp[MLX5DR_BWC_MATCHER_REHASH_BURST_TH];
 	uint16_t burst_th = mlx5dr_bwc_get_burst_th(ctx, queue_id);
 	bool got_comp = *pending_rules >= burst_th;
+	int err = 0;
 	int ret;
 	int i;
 
@@ -203,10 +204,12 @@ mlx5dr_bwc_queue_poll(struct mlx5dr_context *ctx,
 		if (ret) {
 			(*pending_rules) -= ret;
 			for (i = 0; i < ret; i++) {
-				if (unlikely(comp[i].status != RTE_FLOW_OP_SUCCESS))
+				if (unlikely(comp[i].status != RTE_FLOW_OP_SUCCESS)) {
 					DR_LOG(ERR,
 					       "Rehash error: polling queue %d returned completion with error\n",
 					       queue_id);
+					err = -EINVAL;
+				}
 			}
 			queue_full = false;
 		}
@@ -214,7 +217,7 @@ mlx5dr_bwc_queue_poll(struct mlx5dr_context *ctx,
 		got_comp = !!ret;
 	}
 
-	return 0;
+	return err;
 }
 
 static void
@@ -632,7 +635,7 @@ mlx5dr_bwc_matcher_move_all(struct mlx5dr_bwc_matcher *bwc_matcher)
 				ret = mlx5dr_matcher_resize_rule_move(bwc_matcher->matcher,
 								      bwc_rules[i]->rule,
 								      &rule_attr);
-				if (ret) {
+				if (unlikely(ret)) {
 					DR_LOG(ERR, "Moving BWC rule failed during rehash - %d",
 					       ret);
 					rte_errno = ENOMEM;
@@ -643,8 +646,12 @@ mlx5dr_bwc_matcher_move_all(struct mlx5dr_bwc_matcher *bwc_matcher)
 				pending_rules[i]++;
 				bwc_rules[i] = LIST_NEXT(bwc_rules[i], next);
 
-				mlx5dr_bwc_queue_poll(ctx, rule_attr.queue_id,
-						      &pending_rules[i], false);
+				ret = mlx5dr_bwc_queue_poll(ctx, rule_attr.queue_id,
+							    &pending_rules[i], false);
+				if (unlikely(ret)) {
+					rte_errno = EINVAL;
+					goto free_bwc_rules;
+				}
 			}
 		}
 	} while (!all_done);
@@ -654,8 +661,12 @@ mlx5dr_bwc_matcher_move_all(struct mlx5dr_bwc_matcher *bwc_matcher)
 		if (pending_rules[i]) {
 			uint16_t queue_id = mlx5dr_bwc_get_queue_id(ctx, i);
 			mlx5dr_send_engine_flush_queue(&ctx->send_queue[queue_id]);
-			mlx5dr_bwc_queue_poll(ctx, queue_id,
-					      &pending_rules[i], true);
+			ret = mlx5dr_bwc_queue_poll(ctx, queue_id,
+						    &pending_rules[i], true);
+			if (unlikely(ret)) {
+				rte_errno = EINVAL;
+				goto free_bwc_rules;
+			}
 		}
 	}
 
-- 
2.25.1


From 21e8a36c3b8d4858461437e8791852caded1efe3 Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn@nvidia.com>
Date: Sat, 16 Mar 2024 03:31:08 +0200
Subject: [PATCH 5/7] net/mlx5/hws: bwc - go through all the resized matchers

When destroying resizable matcher, we iterate through the list
of previous src matchers and free all their action_ste resources.
Now that we're supporting growing action template sizes, we also
need to support a case when each new dst matcher that we resize
into might have bigger max_ste.
This includes a case where old matchers in the list would have
max_ste 0, while they've been resized into a matcher that has
max_ste > 0.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_matcher.c | 23 +++++++++++++----------
 drivers/net/mlx5/hws/mlx5dr_matcher.h |  1 +
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index aa5ab96d23..6a939eb031 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -724,6 +724,7 @@ mlx5dr_matcher_resize_init(struct mlx5dr_matcher *src_matcher)
 		return rte_errno;
 	}
 
+	resize_data->max_stes = src_matcher->action_ste.max_stes;
 	resize_data->ste = src_matcher->action_ste.ste;
 	resize_data->stc = src_matcher->action_ste.stc;
 	resize_data->action_ste_rtc_0 = src_matcher->action_ste.rtc_0;
@@ -752,23 +753,25 @@ mlx5dr_matcher_resize_uninit(struct mlx5dr_matcher *matcher)
 {
 	struct mlx5dr_matcher_resize_data *resize_data;
 
-	if (!mlx5dr_matcher_is_resizable(matcher) ||
-	    !matcher->action_ste.max_stes)
+	if (!mlx5dr_matcher_is_resizable(matcher))
 		return;
 
 	while (!LIST_EMPTY(&matcher->resize_data)) {
 		resize_data = LIST_FIRST(&matcher->resize_data);
 		LIST_REMOVE(resize_data, next);
 
-		mlx5dr_action_free_single_stc(matcher->tbl->ctx,
-					      matcher->tbl->type,
-					      &resize_data->stc);
+		if (resize_data->max_stes) {
+			mlx5dr_action_free_single_stc(matcher->tbl->ctx,
+						matcher->tbl->type,
+						&resize_data->stc);
+
+			if (matcher->tbl->type == MLX5DR_TABLE_TYPE_FDB)
+				mlx5dr_cmd_destroy_obj(resize_data->action_ste_rtc_1);
+			mlx5dr_cmd_destroy_obj(resize_data->action_ste_rtc_0);
+			if (resize_data->action_ste_pool)
+				mlx5dr_pool_destroy(resize_data->action_ste_pool);
+		}
 
-		if (matcher->tbl->type == MLX5DR_TABLE_TYPE_FDB)
-			mlx5dr_cmd_destroy_obj(resize_data->action_ste_rtc_1);
-		mlx5dr_cmd_destroy_obj(resize_data->action_ste_rtc_0);
-		if (resize_data->action_ste_pool)
-			mlx5dr_pool_destroy(resize_data->action_ste_pool);
 		simple_free(resize_data);
 	}
 }
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.h b/drivers/net/mlx5/hws/mlx5dr_matcher.h
index caff403a38..ca6a5298d9 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.h
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.h
@@ -67,6 +67,7 @@ struct mlx5dr_matcher_resize_data {
 	struct mlx5dr_devx_obj *action_ste_rtc_0;
 	struct mlx5dr_devx_obj *action_ste_rtc_1;
 	struct mlx5dr_pool *action_ste_pool;
+	uint8_t max_stes;
 	LIST_ENTRY(mlx5dr_matcher_resize_data) next;
 };
 
-- 
2.25.1


From e0073e5f4df80951f0e6c5ecba9f2f84b74fad68 Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn@nvidia.com>
Date: Sun, 17 Mar 2024 00:37:19 +0200
Subject: [PATCH 6/7] net/mlx5/hws: bwc - reorg rule resize struct

Reorganize struct mlx5dr_rule_resize_info fields to
reduce its size in memory.

Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_rule.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.h b/drivers/net/mlx5/hws/mlx5dr_rule.h
index bc542eb543..33779ade48 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.h
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.h
@@ -39,11 +39,11 @@ struct mlx5dr_rule_match_tag {
 };
 
 struct mlx5dr_rule_resize_info {
-	uint8_t state;
+	struct mlx5dr_pool *action_ste_pool;
 	uint32_t rtc_0;
 	uint32_t rtc_1;
 	uint32_t rule_idx;
-	struct mlx5dr_pool *action_ste_pool;
+	uint8_t state;
 	uint8_t ctrl_seg[MLX5DR_WQE_SZ_GTA_CTRL]; /* Ctrl segment of STE: 48 bytes */
 	uint8_t data_seg[MLX5DR_WQE_SZ_GTA_DATA]; /* Data segment of STE: 64 bytes */
 };
-- 
2.25.1


From 01109aa3966820f1eec38e6f72385d9adcf58eed Mon Sep 17 00:00:00 2001
From: Yevgeny Kliteynik <kliteyn@nvidia.com>
Date: Sun, 17 Mar 2024 01:26:04 +0200
Subject: [PATCH 7/7] net/mlx5/hws: bwc - fix deleting action stes

If resized rule is deleted, the delete process also includes clearing
the rule's resize_info. The problem is, this resize_info contains info
on the action STEs that are freed onle when the completion of rule
deletion received.
This means that resize_info should be freed only after action STEs.

This patch separates rule's resize_info saving and clearing into a
separate calls. Saving of resize_info is done similar to saving of
delete info (same as it was before this patch), and clearing is done
only after mlx5dr_rule_free_action_ste_idx.

Additional issue that is fixed here is using the right max_stes when
calculating which STEs should be freed - max_stes is now saved as part
of resize_info.

Fixes: 762feceb8294 ("net/mlx5/hws: support resizable matchers")
Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_rule.c | 44 ++++++++++++++++++------------
 drivers/net/mlx5/hws/mlx5dr_rule.h |  3 ++
 drivers/net/mlx5/hws/mlx5dr_send.c |  8 ++++--
 3 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c b/drivers/net/mlx5/hws/mlx5dr_rule.c
index 550f00a4c1..74abb38163 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.c
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -157,6 +157,9 @@ static void
 mlx5dr_rule_save_resize_info(struct mlx5dr_rule *rule,
 			     struct mlx5dr_send_ste_attr *ste_attr)
 {
+	if (likely(!mlx5dr_matcher_is_resizable(rule->matcher)))
+		return;
+
 	rule->resize_info = simple_calloc(1, sizeof(*rule->resize_info));
 	if (unlikely(!rule->resize_info)) {
 		assert(rule->resize_info);
@@ -168,14 +171,16 @@ mlx5dr_rule_save_resize_info(struct mlx5dr_rule *rule,
 	memcpy(rule->resize_info->data_seg, ste_attr->wqe_data,
 	       sizeof(rule->resize_info->data_seg));
 
+	rule->resize_info->max_stes = rule->matcher->action_ste.max_stes;
 	rule->resize_info->action_ste_pool = rule->matcher->action_ste.max_stes ?
 					     rule->matcher->action_ste.pool :
 					     NULL;
 }
 
-static void mlx5dr_rule_clear_resize_info(struct mlx5dr_rule *rule)
+void mlx5dr_rule_clear_resize_info(struct mlx5dr_rule *rule)
 {
-	if (rule->resize_info) {
+	if (unlikely(mlx5dr_matcher_is_resizable(rule->matcher) &&
+		     rule->resize_info)) {
 		simple_free(rule->resize_info);
 		rule->resize_info = NULL;
 	}
@@ -220,8 +225,6 @@ mlx5dr_rule_save_delete_info(struct mlx5dr_rule *rule,
 			memcpy(&rule->tag.match, ste_attr->wqe_data->tag, MLX5DR_MATCH_TAG_SZ);
 		return;
 	}
-
-	mlx5dr_rule_save_resize_info(rule, ste_attr);
 }
 
 static void
@@ -231,11 +234,6 @@ mlx5dr_rule_clear_delete_info(struct mlx5dr_rule *rule)
 		simple_free(rule->tag_ptr);
 		return;
 	}
-
-	if (unlikely(mlx5dr_matcher_is_resizable(rule->matcher))) {
-		mlx5dr_rule_clear_resize_info(rule);
-		return;
-	}
 }
 
 static void
@@ -288,19 +286,26 @@ void mlx5dr_rule_free_action_ste_idx(struct mlx5dr_rule *rule)
 {
 	struct mlx5dr_matcher *matcher = rule->matcher;
 	struct mlx5dr_pool *pool;
+	uint8_t max_stes;
 
 	if (rule->action_ste_idx > -1 &&
 	    !matcher->attr.optimize_using_rule_idx &&
 	    !mlx5dr_matcher_is_insert_by_idx(matcher)) {
 		struct mlx5dr_pool_chunk ste = {0};
 
+		if (unlikely(mlx5dr_matcher_is_resizable(matcher))) {
+			/* Free the original action pool if rule was resized */
+			max_stes = rule->resize_info->max_stes;
+			pool = rule->resize_info->action_ste_pool;
+		} else {
+			max_stes = matcher->action_ste.max_stes;
+			pool = matcher->action_ste.pool;
+		}
+
 		/* This release is safe only when the rule match part was deleted */
-		ste.order = rte_log2_u32(matcher->action_ste.max_stes);
+		ste.order = rte_log2_u32(max_stes);
 		ste.offset = rule->action_ste_idx;
 
-		/* Free the original action pool if rule was resized */
-		pool = mlx5dr_matcher_is_resizable(matcher) ? rule->resize_info->action_ste_pool :
-							      matcher->action_ste.pool;
 		mlx5dr_pool_chunk_free(pool, &ste);
 	}
 }
@@ -442,9 +447,7 @@ static int mlx5dr_rule_create_hws_fw_wqe(struct mlx5dr_rule *rule,
 	/* Send WQEs to FW */
 	mlx5dr_send_stes_fw(queue, &ste_attr);
 
-	/* Backup TAG on the rule for deletion, and save ctrl/data
-	 * segments to be used when resizing the matcher.
-	 */
+	/* Backup TAG on the rule for deletion */
 	mlx5dr_rule_save_delete_info(rule, &ste_attr);
 	mlx5dr_send_engine_inc_rule(queue);
 
@@ -569,8 +572,10 @@ static int mlx5dr_rule_create_hws(struct mlx5dr_rule *rule,
 	/* Backup TAG on the rule for deletion and resize info for
 	 * moving rules to a new matcher, only after insertion.
 	 */
-	if (!is_update)
+	if (!is_update) {
 		mlx5dr_rule_save_delete_info(rule, &ste_attr);
+		mlx5dr_rule_save_resize_info(rule, &ste_attr);
+	}
 
 	mlx5dr_send_engine_inc_rule(queue);
 
@@ -595,9 +600,12 @@ static void mlx5dr_rule_destroy_failed_hws(struct mlx5dr_rule *rule,
 	/* Rule failed now we can safely release action STEs */
 	mlx5dr_rule_free_action_ste_idx(rule);
 
-	/* Clear complex tag or info that was saved for matcher resizing */
+	/* Clear complex tag */
 	mlx5dr_rule_clear_delete_info(rule);
 
+	/* Clear info that was saved for resizing */
+	mlx5dr_rule_clear_resize_info(rule);
+
 	/* If a rule that was indicated as burst (need to trigger HW) has failed
 	 * insertion we won't ring the HW as nothing is being written to the WQ.
 	 * In such case update the last WQE and ring the HW with that work
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.h b/drivers/net/mlx5/hws/mlx5dr_rule.h
index 33779ade48..3617fd59dd 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.h
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.h
@@ -44,6 +44,7 @@ struct mlx5dr_rule_resize_info {
 	uint32_t rtc_1;
 	uint32_t rule_idx;
 	uint8_t state;
+	uint8_t max_stes;
 	uint8_t ctrl_seg[MLX5DR_WQE_SZ_GTA_CTRL]; /* Ctrl segment of STE: 48 bytes */
 	uint8_t data_seg[MLX5DR_WQE_SZ_GTA_DATA]; /* Data segment of STE: 64 bytes */
 };
@@ -74,6 +75,8 @@ int mlx5dr_rule_move_hws_add(struct mlx5dr_rule *rule,
 
 bool mlx5dr_rule_move_in_progress(struct mlx5dr_rule *rule);
 
+void mlx5dr_rule_clear_resize_info(struct mlx5dr_rule *rule);
+
 int mlx5dr_rule_create_root_no_comp(struct mlx5dr_rule *rule,
 				    const struct rte_flow_item items[],
 				    uint8_t num_actions,
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c b/drivers/net/mlx5/hws/mlx5dr_send.c
index 0120f03a48..3022c50260 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.c
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -523,9 +523,13 @@ static void mlx5dr_send_engine_update_rule(struct mlx5dr_send_engine *queue,
 			 */
 			priv->rule->status++;
 			*status = RTE_FLOW_OP_SUCCESS;
-			/* Rule was deleted now we can safely release action STEs */
-			if (priv->rule->status == MLX5DR_RULE_STATUS_DELETED)
+			/* Rule was deleted now we can safely release action STEs
+			 * and clear resize info
+			 */
+			if (priv->rule->status == MLX5DR_RULE_STATUS_DELETED) {
 				mlx5dr_rule_free_action_ste_idx(priv->rule);
+				mlx5dr_rule_clear_resize_info(priv->rule);
+			}
 		}
 	}
 }
-- 
2.25.1

[PATCH v2 26/34] net/mlx5/hws: fix bug in matcher disconnect error flow

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

If error happens during disconnect of the first matcher in the
list, the matcher should be reconnected back as the first matcher.

Fixes: b81f95ca770d ("net/mlx5/hws: support default miss table")
Cc: stable@dpdk.org
Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_matcher.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index 79e7401dfd..2a84145566 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -340,7 +340,7 @@ static int mlx5dr_matcher_disconnect(struct mlx5dr_matcher *matcher)
 	return 0;
 
 matcher_reconnect:
-	if (LIST_EMPTY(&tbl->head))
+	if (LIST_EMPTY(&tbl->head) || prev_matcher == matcher)
 		LIST_INSERT_HEAD(&matcher->tbl->head, matcher, next);
 	else
 		LIST_INSERT_AFTER(prev_matcher, matcher, next);
-- 
2.25.1

[PATCH v3 1/7] net/mlx5/hws: fix bug in matcher disconnect error flow

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

If error happens during disconnect of the first matcher in the
list, the matcher should be reconnected back as the first matcher.

Fixes: b81f95ca770d ("net/mlx5/hws: support default miss table")
Cc: stable@dpdk.org
Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_matcher.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index 79e7401dfd..2a84145566 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -340,7 +340,7 @@ static int mlx5dr_matcher_disconnect(struct mlx5dr_matcher *matcher)
 	return 0;
 
 matcher_reconnect:
-	if (LIST_EMPTY(&tbl->head))
+	if (LIST_EMPTY(&tbl->head) || prev_matcher == matcher)
 		LIST_INSERT_HEAD(&matcher->tbl->head, matcher, next);
 	else
 		LIST_INSERT_AFTER(prev_matcher, matcher, next);
-- 
2.25.1

[PATCH v4 1/7] net/mlx5/hws: fix bug in matcher disconnect error flow

[Maayan Kashani]

From: Yevgeny Kliteynik <kliteyn@nvidia.com>

If error happens during disconnect of the first matcher in the
list, the matcher should be reconnected back as the first matcher.

Fixes: b81f95ca770d ("net/mlx5/hws: support default miss table")
Cc: stable@dpdk.org
Signed-off-by: Yevgeny Kliteynik <kliteyn@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
 drivers/net/mlx5/hws/mlx5dr_matcher.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index 79e7401dfd..2a84145566 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -340,7 +340,7 @@ static int mlx5dr_matcher_disconnect(struct mlx5dr_matcher *matcher)
 	return 0;
 
 matcher_reconnect:
-	if (LIST_EMPTY(&tbl->head))
+	if (LIST_EMPTY(&tbl->head) || prev_matcher == matcher)
 		LIST_INSERT_HEAD(&matcher->tbl->head, matcher, next);
 	else
 		LIST_INSERT_AFTER(prev_matcher, matcher, next);
-- 
2.21.0