patch 'net/mlx5: fix race condition in counter pool resizing' has been queued to stable release 20.11.7

[luca.boccassi@gmail.com]

Hi,

FYI, your patch has been queued to stable release 20.11.7

Note it hasn't been pushed to http://dpdk.org/browse/dpdk-stable yet.
It will be pushed if I get no objections before 11/19/22. So please
shout if anyone has objections.

Also note that after the patch there's a diff of the upstream commit vs the
patch applied to the branch. This will indicate if there was any rebasing
needed to apply to the stable branch. If there were code changes for rebasing
(ie: not only metadata diffs), please double check that the rebase was
correctly done.

Queued patches are on a temporary branch at:
https://github.com/kevintraynor/dpdk-stable

This queued commit can be viewed at:
https://github.com/kevintraynor/dpdk-stable/commit/4951ff74e90470dfd51af5fe148e1ce9e674c7bb

Thanks.

Luca Boccassi

---
From 4951ff74e90470dfd51af5fe148e1ce9e674c7bb Mon Sep 17 00:00:00 2001
From: Michael Baum <michaelba@nvidia.com>
Date: Mon, 31 Oct 2022 18:08:20 +0200
Subject: [PATCH] net/mlx5: fix race condition in counter pool resizing

[ upstream commit a94e89e47b59ebaf84246bbb34c06e1a004cde8a ]

Counter management structure has array of counter pools. This array is
invalid in management structure initialization and grows on demand.

The resizing include:
1. Allocate memory for the new size.
2. Copy the existing data to the new memory.
3. Move the pointer to the new memory.
4. Free the old memory.

The third step can be performed before for this function, and compiler
may do that, but another thread might read the pointer before coping and
read invalid data or even crash.

This patch allocates memory for this array once in management structure
initialization and limit the counters number by 16M.

Fixes: 3aa279157fa0 ("net/mlx5: synchronize flow counter pool creation")

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c            | 27 ++++++++++++---
 drivers/net/mlx5/mlx5.h            |  5 +--
 drivers/net/mlx5/mlx5_flow.c       | 22 ++++++-------
 drivers/net/mlx5/mlx5_flow_dv.c    | 53 +++++-------------------------
 drivers/net/mlx5/mlx5_flow_verbs.c | 23 +++----------
 5 files changed, 50 insertions(+), 80 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0b82969b4d..90985479de 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -447,22 +447,38 @@ mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh)
  *
  * @param[in] sh
  *   Pointer to mlx5_dev_ctx_shared object to free
+ *
+ * @return
+ *   0 on success, otherwise negative errno value and rte_errno is set.
  */
-static void
+static int
 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh)
 {
 	int i;
+	void *pools;
 
+	pools = mlx5_malloc(MLX5_MEM_ZERO,
+				sizeof(struct mlx5_flow_counter_pool *) *
+				MLX5_COUNTER_POOLS_MAX_NUM,
+				0, SOCKET_ID_ANY);
+	if (!pools) {
+		DRV_LOG(ERR,
+			"Counter management allocation was failed.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
 	memset(&sh->cmng, 0, sizeof(sh->cmng));
 	TAILQ_INIT(&sh->cmng.flow_counters);
 	sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET;
 	sh->cmng.max_id = -1;
 	sh->cmng.last_pool_idx = POOL_IDX_INVALID;
+	sh->cmng.pools = pools;
 	rte_spinlock_init(&sh->cmng.pool_update_sl);
 	for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) {
 		TAILQ_INIT(&sh->cmng.counters[i]);
 		rte_spinlock_init(&sh->cmng.csl[i]);
 	}
+	return 0;
 }
 
 /**
@@ -520,8 +536,7 @@ mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh)
 					claim_zero
 					 (mlx5_flow_os_destroy_flow_action
 					  (cnt->action));
-				if (fallback && MLX5_POOL_GET_CNT
-				    (pool, j)->dcs_when_free)
+				if (fallback && cnt->dcs_when_free)
 					claim_zero(mlx5_devx_cmd_destroy
 						   (cnt->dcs_when_free));
 			}
@@ -1002,8 +1017,12 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 		err = rte_errno;
 		goto error;
 	}
+	err = mlx5_flow_counters_mng_init(sh);
+	if (err) {
+		DRV_LOG(ERR, "Fail to initialize counters manage.");
+		goto error;
+	}
 	mlx5_flow_aging_init(sh);
-	mlx5_flow_counters_mng_init(sh);
 	mlx5_flow_ipool_create(sh, config);
 	/* Add device to memory callback list. */
 	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 6c85c4e2db..69fb34826d 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -280,9 +280,10 @@ struct mlx5_lb_ctx {
 	uint16_t refcnt; /* Reference count for representors. */
 };
 
+#define MLX5_COUNTER_POOLS_MAX_NUM (1 << 15)
 #define MLX5_COUNTERS_PER_POOL 512
 #define MLX5_MAX_PENDING_QUERIES 4
-#define MLX5_CNT_CONTAINER_RESIZE 64
+#define MLX5_CNT_MR_ALLOC_BULK 64
 #define MLX5_CNT_SHARED_OFFSET 0x80000000
 #define IS_SHARED_CNT(cnt) (!!((cnt) & MLX5_CNT_SHARED_OFFSET))
 #define IS_BATCH_CNT(cnt) (((cnt) & (MLX5_CNT_SHARED_OFFSET - 1)) >= \
@@ -442,7 +443,6 @@ TAILQ_HEAD(mlx5_counter_pools, mlx5_flow_counter_pool);
 /* Counter global management structure. */
 struct mlx5_flow_counter_mng {
 	volatile uint16_t n_valid; /* Number of valid pools. */
-	uint16_t n; /* Number of pools. */
 	uint16_t last_pool_idx; /* Last used pool index */
 	int min_id; /* The minimum counter ID in the pools. */
 	int max_id; /* The maximum counter ID in the pools. */
@@ -514,6 +514,7 @@ struct mlx5_aso_age_action {
 };
 
 #define MLX5_ASO_AGE_ACTIONS_PER_POOL 512
+#define MLX5_ASO_AGE_CONTAINER_RESIZE 64
 
 struct mlx5_aso_age_pool {
 	struct mlx5_devx_obj *flow_hit_aso_obj;
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 43fa595be1..f0c90a314b 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -6647,7 +6647,7 @@ mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
 {
 	struct mlx5_counter_stats_mem_mng *mem_mng;
 	volatile struct flow_counter_stats *raw_data;
-	int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;
+	int raws_n = MLX5_CNT_MR_ALLOC_BULK + MLX5_MAX_PENDING_QUERIES;
 	int size = (sizeof(struct flow_counter_stats) *
 			MLX5_COUNTERS_PER_POOL +
 			sizeof(struct mlx5_counter_stats_raw)) * raws_n +
@@ -6685,7 +6685,7 @@ mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
 	}
 	for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
 		LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
-				 mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE + i,
+				 mem_mng->raws + MLX5_CNT_MR_ALLOC_BULK + i,
 				 next);
 	LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
 	sh->cmng.mem_mng = mem_mng;
@@ -6709,14 +6709,13 @@ mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
 {
 	struct mlx5_flow_counter_mng *cmng = &sh->cmng;
 	/* Resize statistic memory once used out. */
-	if (!(pool->index % MLX5_CNT_CONTAINER_RESIZE) &&
+	if (!(pool->index % MLX5_CNT_MR_ALLOC_BULK) &&
 	    mlx5_flow_create_counter_stat_mem_mng(sh)) {
 		DRV_LOG(ERR, "Cannot resize counter stat mem.");
 		return -1;
 	}
 	rte_spinlock_lock(&pool->sl);
-	pool->raw = cmng->mem_mng->raws + pool->index %
-		    MLX5_CNT_CONTAINER_RESIZE;
+	pool->raw = cmng->mem_mng->raws + pool->index % MLX5_CNT_MR_ALLOC_BULK;
 	rte_spinlock_unlock(&pool->sl);
 	pool->raw_hw = NULL;
 	return 0;
@@ -6758,13 +6757,13 @@ void
 mlx5_flow_query_alarm(void *arg)
 {
 	struct mlx5_dev_ctx_shared *sh = arg;
-	int ret;
-	uint16_t pool_index = sh->cmng.pool_index;
 	struct mlx5_flow_counter_mng *cmng = &sh->cmng;
+	uint16_t pool_index = cmng->pool_index;
 	struct mlx5_flow_counter_pool *pool;
 	uint16_t n_valid;
+	int ret;
 
-	if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
+	if (cmng->pending_queries >= MLX5_MAX_PENDING_QUERIES)
 		goto set_alarm;
 	rte_spinlock_lock(&cmng->pool_update_sl);
 	pool = cmng->pools[pool_index];
@@ -6776,8 +6775,7 @@ mlx5_flow_query_alarm(void *arg)
 	if (pool->raw_hw)
 		/* There is a pool query in progress. */
 		goto set_alarm;
-	pool->raw_hw =
-		LIST_FIRST(&sh->cmng.free_stat_raws);
+	pool->raw_hw = LIST_FIRST(&cmng->free_stat_raws);
 	if (!pool->raw_hw)
 		/* No free counter statistics raw memory. */
 		goto set_alarm;
@@ -6803,12 +6801,12 @@ mlx5_flow_query_alarm(void *arg)
 		goto set_alarm;
 	}
 	LIST_REMOVE(pool->raw_hw, next);
-	sh->cmng.pending_queries++;
+	cmng->pending_queries++;
 	pool_index++;
 	if (pool_index >= n_valid)
 		pool_index = 0;
 set_alarm:
-	sh->cmng.pool_index = pool_index;
+	cmng->pool_index = pool_index;
 	mlx5_set_query_alarm(sh);
 }
 
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index f25ee9f92f..1705e2a1f0 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -4650,7 +4650,7 @@ flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
 
 	/* Decrease to original index and clear shared bit. */
 	idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1);
-	MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cmng->n);
+	MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < MLX5_COUNTER_POOLS_MAX_NUM);
 	pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL];
 	MLX5_ASSERT(pool);
 	if (ppool)
@@ -4726,39 +4726,6 @@ out:
 	return pool;
 }
 
-/**
- * Resize a counter container.
- *
- * @param[in] dev
- *   Pointer to the Ethernet device structure.
- *
- * @return
- *   0 on success, otherwise negative errno value and rte_errno is set.
- */
-static int
-flow_dv_container_resize(struct rte_eth_dev *dev)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
-	void *old_pools = cmng->pools;
-	uint32_t resize = cmng->n + MLX5_CNT_CONTAINER_RESIZE;
-	uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
-	void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
-
-	if (!pools) {
-		rte_errno = ENOMEM;
-		return -ENOMEM;
-	}
-	if (old_pools)
-		memcpy(pools, old_pools, cmng->n *
-				       sizeof(struct mlx5_flow_counter_pool *));
-	cmng->n = resize;
-	cmng->pools = pools;
-	if (old_pools)
-		mlx5_free(old_pools);
-	return 0;
-}
-
 /**
  * Query a devx flow counter.
  *
@@ -4810,8 +4777,6 @@ _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
  *   The devX counter handle.
  * @param[in] age
  *   Whether the pool is for counter that was allocated for aging.
- * @param[in/out] cont_cur
- *   Pointer to the container pointer, it will be update in pool resize.
  *
  * @return
  *   The pool container pointer on success, NULL otherwise and rte_errno is set.
@@ -4823,9 +4788,14 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_counter_pool *pool;
 	struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
-	bool fallback = priv->sh->cmng.counter_fallback;
+	bool fallback = cmng->counter_fallback;
 	uint32_t size = sizeof(*pool);
 
+	if (cmng->n_valid == MLX5_COUNTER_POOLS_MAX_NUM) {
+		DRV_LOG(ERR, "All counter is in used, try again later.");
+		rte_errno = EAGAIN;
+		return NULL;
+	}
 	size += MLX5_COUNTERS_PER_POOL * MLX5_CNT_SIZE;
 	size += (!age ? 0 : MLX5_COUNTERS_PER_POOL * MLX5_AGE_SIZE);
 	pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
@@ -4844,11 +4814,6 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
 	pool->time_of_last_age_check = MLX5_CURR_TIME_SEC;
 	rte_spinlock_lock(&cmng->pool_update_sl);
 	pool->index = cmng->n_valid;
-	if (pool->index == cmng->n && flow_dv_container_resize(dev)) {
-		mlx5_free(pool);
-		rte_spinlock_unlock(&cmng->pool_update_sl);
-		return NULL;
-	}
 	cmng->pools[pool->index] = pool;
 	cmng->n_valid++;
 	if (unlikely(fallback)) {
@@ -9533,7 +9498,7 @@ flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
 }
 
 /**
- * Resize the ASO age pools array by MLX5_CNT_CONTAINER_RESIZE pools.
+ * Resize the ASO age pools array by MLX5_ASO_AGE_CONTAINER_RESIZE pools.
  *
  * @param[in] dev
  *   Pointer to the Ethernet device structure.
@@ -9547,7 +9512,7 @@ flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_aso_age_mng *mng = priv->sh->aso_age_mng;
 	void *old_pools = mng->pools;
-	uint32_t resize = mng->n + MLX5_CNT_CONTAINER_RESIZE;
+	uint32_t resize = mng->n + MLX5_ASO_AGE_CONTAINER_RESIZE;
 	uint32_t mem_size = sizeof(struct mlx5_aso_age_pool *) * resize;
 	void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY);
 
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 6a755e7c36..340cea3a6b 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -274,27 +274,14 @@ flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 			break;
 	}
 	if (!cnt) {
-		struct mlx5_flow_counter_pool **pools;
 		uint32_t size;
 
-		if (n_valid == cmng->n) {
-			/* Resize the container pool array. */
-			size = sizeof(struct mlx5_flow_counter_pool *) *
-				     (n_valid + MLX5_CNT_CONTAINER_RESIZE);
-			pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0,
-					    SOCKET_ID_ANY);
-			if (!pools)
-				return 0;
-			if (n_valid) {
-				memcpy(pools, cmng->pools,
-				       sizeof(struct mlx5_flow_counter_pool *) *
-				       n_valid);
-				mlx5_free(cmng->pools);
-			}
-			cmng->pools = pools;
-			cmng->n += MLX5_CNT_CONTAINER_RESIZE;
+		if (n_valid == MLX5_COUNTER_POOLS_MAX_NUM) {
+			DRV_LOG(ERR, "All counter is in used, try again later.");
+			rte_errno = EAGAIN;
+			return 0;
 		}
-		/* Allocate memory for new pool*/
+		/* Allocate memory for new pool */
 		size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL;
 		pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY);
 		if (!pool)
-- 
2.34.1

---
  Diff of the applied patch vs upstream commit (please double-check if non-empty:
---
--- -	2022-11-17 23:07:56.008708071 +0000
+++ 0012-net-mlx5-fix-race-condition-in-counter-pool-resizing.patch	2022-11-17 23:07:55.472329917 +0000
@@ -1 +1 @@
-From a94e89e47b59ebaf84246bbb34c06e1a004cde8a Mon Sep 17 00:00:00 2001
+From 4951ff74e90470dfd51af5fe148e1ce9e674c7bb Mon Sep 17 00:00:00 2001
@@ -5,0 +6,2 @@
+[ upstream commit a94e89e47b59ebaf84246bbb34c06e1a004cde8a ]
+
@@ -23 +24,0 @@
-Cc: stable@dpdk.org
@@ -28,3 +29,3 @@
- drivers/net/mlx5/mlx5.c            | 28 +++++++++++++---
- drivers/net/mlx5/mlx5.h            |  7 ++--
- drivers/net/mlx5/mlx5_flow.c       | 24 +++++++-------
+ drivers/net/mlx5/mlx5.c            | 27 ++++++++++++---
+ drivers/net/mlx5/mlx5.h            |  5 +--
+ drivers/net/mlx5/mlx5_flow.c       | 22 ++++++-------
@@ -33 +34 @@
- 5 files changed, 52 insertions(+), 83 deletions(-)
+ 5 files changed, 50 insertions(+), 80 deletions(-)
@@ -36 +37 @@
-index 78234b116c..b85a56ec24 100644
+index 0b82969b4d..90985479de 100644
@@ -39 +40 @@
-@@ -561,18 +561,34 @@ mlx5_flow_counter_mode_config(struct rte_eth_dev *dev __rte_unused)
+@@ -447,22 +447,38 @@ mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh)
@@ -51 +52,2 @@
- 	int i, j;
+ 	int i;
++	void *pools;
@@ -53,25 +55,20 @@
- 	if (sh->config.dv_flow_en < 2) {
-+		void *pools;
-+
-+		pools = mlx5_malloc(MLX5_MEM_ZERO,
-+				    sizeof(struct mlx5_flow_counter_pool *) *
-+				    MLX5_COUNTER_POOLS_MAX_NUM,
-+				    0, SOCKET_ID_ANY);
-+		if (!pools) {
-+			DRV_LOG(ERR,
-+				"Counter management allocation was failed.");
-+			rte_errno = ENOMEM;
-+			return -rte_errno;
-+		}
- 		memset(&sh->sws_cmng, 0, sizeof(sh->sws_cmng));
- 		TAILQ_INIT(&sh->sws_cmng.flow_counters);
- 		sh->sws_cmng.min_id = MLX5_CNT_BATCH_OFFSET;
- 		sh->sws_cmng.max_id = -1;
- 		sh->sws_cmng.last_pool_idx = POOL_IDX_INVALID;
-+		sh->sws_cmng.pools = pools;
- 		rte_spinlock_init(&sh->sws_cmng.pool_update_sl);
- 		for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) {
- 			TAILQ_INIT(&sh->sws_cmng.counters[i]);
-@@ -598,6 +614,7 @@ mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh)
- 		sh->hws_max_log_bulk_sz = log_dcs;
- 		sh->hws_max_nb_counters = max_nb_cnts;
++	pools = mlx5_malloc(MLX5_MEM_ZERO,
++				sizeof(struct mlx5_flow_counter_pool *) *
++				MLX5_COUNTER_POOLS_MAX_NUM,
++				0, SOCKET_ID_ANY);
++	if (!pools) {
++		DRV_LOG(ERR,
++			"Counter management allocation was failed.");
++		rte_errno = ENOMEM;
++		return -rte_errno;
++	}
+ 	memset(&sh->cmng, 0, sizeof(sh->cmng));
+ 	TAILQ_INIT(&sh->cmng.flow_counters);
+ 	sh->cmng.min_id = MLX5_CNT_BATCH_OFFSET;
+ 	sh->cmng.max_id = -1;
+ 	sh->cmng.last_pool_idx = POOL_IDX_INVALID;
++	sh->cmng.pools = pools;
+ 	rte_spinlock_init(&sh->cmng.pool_update_sl);
+ 	for (i = 0; i < MLX5_COUNTER_TYPE_MAX; i++) {
+ 		TAILQ_INIT(&sh->cmng.counters[i]);
+ 		rte_spinlock_init(&sh->cmng.csl[i]);
@@ -83 +80 @@
-@@ -655,8 +672,7 @@ mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh)
+@@ -520,8 +536,7 @@ mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh)
@@ -93,3 +90,3 @@
-@@ -1572,8 +1588,12 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
- 		if (err)
- 			goto error;
+@@ -1002,8 +1017,12 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
+ 		err = rte_errno;
+ 		goto error;
@@ -104,3 +101,3 @@
- 	mlx5_flow_ipool_create(sh);
- 	/* Add context to the global device list. */
- 	LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next);
+ 	mlx5_flow_ipool_create(sh, config);
+ 	/* Add device to memory callback list. */
+ 	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
@@ -108 +105 @@
-index c9fcb71b69..cbe2d88b9e 100644
+index 6c85c4e2db..69fb34826d 100644
@@ -111,3 +108,3 @@
-@@ -386,11 +386,10 @@ struct mlx5_hw_q {
- } __rte_cache_aligned;
- 
+@@ -280,9 +280,10 @@ struct mlx5_lb_ctx {
+ 	uint16_t refcnt; /* Reference count for representors. */
+ };
@@ -115,2 +111,0 @@
--
--
@@ -122,0 +118 @@
+ #define IS_SHARED_CNT(cnt) (!!((cnt) & MLX5_CNT_SHARED_OFFSET))
@@ -124,2 +120 @@
- 			   MLX5_CNT_BATCH_OFFSET)
-@@ -549,7 +548,6 @@ TAILQ_HEAD(mlx5_counter_pools, mlx5_flow_counter_pool);
+@@ -442,7 +443,6 @@ TAILQ_HEAD(mlx5_counter_pools, mlx5_flow_counter_pool);
@@ -133 +128 @@
-@@ -621,6 +619,7 @@ struct mlx5_aso_age_action {
+@@ -514,6 +514,7 @@ struct mlx5_aso_age_action {
@@ -142 +137 @@
-index e28587da8e..78c92fe086 100644
+index 43fa595be1..f0c90a314b 100644
@@ -145 +140 @@
-@@ -9061,7 +9061,7 @@ mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
+@@ -6647,7 +6647,7 @@ mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
@@ -154 +149 @@
-@@ -9099,7 +9099,7 @@ mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
+@@ -6685,7 +6685,7 @@ mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
@@ -157 +152 @@
- 		LIST_INSERT_HEAD(&sh->sws_cmng.free_stat_raws,
+ 		LIST_INSERT_HEAD(&sh->cmng.free_stat_raws,
@@ -161,3 +156,3 @@
- 	LIST_INSERT_HEAD(&sh->sws_cmng.mem_mngs, mem_mng, next);
- 	sh->sws_cmng.mem_mng = mem_mng;
-@@ -9123,14 +9123,13 @@ mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
+ 	LIST_INSERT_HEAD(&sh->cmng.mem_mngs, mem_mng, next);
+ 	sh->cmng.mem_mng = mem_mng;
+@@ -6709,14 +6709,13 @@ mlx5_flow_set_counter_stat_mem(struct mlx5_dev_ctx_shared *sh,
@@ -165 +160 @@
- 	struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng;
+ 	struct mlx5_flow_counter_mng *cmng = &sh->cmng;
@@ -180 +175 @@
-@@ -9172,13 +9171,13 @@ void
+@@ -6758,13 +6757,13 @@ void
@@ -185,2 +180,2 @@
--	uint16_t pool_index = sh->sws_cmng.pool_index;
- 	struct mlx5_flow_counter_mng *cmng = &sh->sws_cmng;
+-	uint16_t pool_index = sh->cmng.pool_index;
+ 	struct mlx5_flow_counter_mng *cmng = &sh->cmng;
@@ -192 +187 @@
--	if (sh->sws_cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
+-	if (sh->cmng.pending_queries >= MLX5_MAX_PENDING_QUERIES)
@@ -197 +192 @@
-@@ -9190,8 +9189,7 @@ mlx5_flow_query_alarm(void *arg)
+@@ -6776,8 +6775,7 @@ mlx5_flow_query_alarm(void *arg)
@@ -202 +197 @@
--		LIST_FIRST(&sh->sws_cmng.free_stat_raws);
+-		LIST_FIRST(&sh->cmng.free_stat_raws);
@@ -207 +202 @@
-@@ -9217,12 +9215,12 @@ mlx5_flow_query_alarm(void *arg)
+@@ -6803,12 +6801,12 @@ mlx5_flow_query_alarm(void *arg)
@@ -211 +206 @@
--	sh->sws_cmng.pending_queries++;
+-	sh->cmng.pending_queries++;
@@ -217 +212 @@
--	sh->sws_cmng.pool_index = pool_index;
+-	sh->cmng.pool_index = pool_index;
@@ -222,9 +216,0 @@
-@@ -9753,7 +9751,7 @@ mlx5_flow_dev_dump_sh_all(struct rte_eth_dev *dev,
- 	}
- 
- 	/* get counter */
--	MLX5_ASSERT(cmng->n_valid <= cmng->n);
-+	MLX5_ASSERT(cmng->n_valid <= MLX5_COUNTER_POOLS_MAX_NUM);
- 	max = MLX5_COUNTERS_PER_POOL * cmng->n_valid;
- 	for (j = 1; j <= max; j++) {
- 		action = NULL;
@@ -232 +218 @@
-index 1e52278191..e77cbb862b 100644
+index f25ee9f92f..1705e2a1f0 100644
@@ -235 +221 @@
-@@ -6091,7 +6091,7 @@ flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
+@@ -4650,7 +4650,7 @@ flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
@@ -244 +230 @@
-@@ -6167,39 +6167,6 @@ out:
+@@ -4726,39 +4726,6 @@ out:
@@ -261 +247 @@
--	struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng;
+-	struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
@@ -284 +270 @@
-@@ -6251,8 +6218,6 @@ _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
+@@ -4810,8 +4777,6 @@ _flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
@@ -293 +279 @@
-@@ -6264,9 +6229,14 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
+@@ -4823,9 +4788,14 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
@@ -296,2 +282,2 @@
- 	struct mlx5_flow_counter_mng *cmng = &priv->sh->sws_cmng;
--	bool fallback = priv->sh->sws_cmng.counter_fallback;
+ 	struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng;
+-	bool fallback = priv->sh->cmng.counter_fallback;
@@ -309 +295 @@
-@@ -6285,11 +6255,6 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
+@@ -4844,11 +4814,6 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
@@ -321 +307 @@
-@@ -12511,7 +12476,7 @@ flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
+@@ -9533,7 +9498,7 @@ flow_dv_aso_age_release(struct rte_eth_dev *dev, uint32_t age_idx)
@@ -330 +316 @@
-@@ -12525,7 +12490,7 @@ flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
+@@ -9547,7 +9512,7 @@ flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev)
@@ -340 +326 @@
-index 81a33ddf09..4bca685674 100644
+index 6a755e7c36..340cea3a6b 100644
@@ -343 +329 @@
-@@ -232,27 +232,14 @@ flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t id __rte_unused)
+@@ -274,27 +274,14 @@ flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)