[dpdk-dev] [PATCH 0/4] net/mlx5: add indexed pool local cache

[dpdk-dev] [PATCH 0/4] net/mlx5: add indexed pool local cache

[Suanming Mou]

For object which wants efficient index allocate and free, local
cache will be very helpful.

Two level cache is added, one as local and another as global.
The global cache is able to save all the allocated index. That
means all the allocated index will not be freed. Once the local
cache is full, the extra index will be flushed to the global
cache. Once local cache is empty, first try to fetch more index
from global, if global is still empty, allocate new trunk and
more index.


Suanming Mou (4):
  net/mlx5: add index allocate with up limit
  net/mlx5: add indexed pool local cache
  net/mlx5: add index pool cache flush
  net/mlx5: replace flow list with index pool

 drivers/net/mlx5/linux/mlx5_os.c |  11 +
 drivers/net/mlx5/mlx5.c          |   3 +-
 drivers/net/mlx5/mlx5.h          |   4 +-
 drivers/net/mlx5/mlx5_flow.c     | 111 ++++------
 drivers/net/mlx5/mlx5_flow.h     |   1 +
 drivers/net/mlx5/mlx5_flow_dv.c  |   5 +
 drivers/net/mlx5/mlx5_trigger.c  |   8 +-
 drivers/net/mlx5/mlx5_utils.c    | 337 ++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_utils.h    |  69 ++++++-
 9 files changed, 463 insertions(+), 86 deletions(-)

-- 
2.25.1

[dpdk-dev] [PATCH 1/4] net/mlx5: add index allocate with up limit

[Suanming Mou]

The index pool can be used as ID allocator. In the ID allocator case,
to support the maximum ID can be allocated is very useful since some
registers only want limited bits of ID.

This patch add the maximum ID configurable for the index pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 18fe23e4fb..bf2b2ebc72 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index b54517c6df..15870e14c2 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
-- 
2.25.1

[dpdk-dev] [PATCH 2/4] net/mlx5: add indexed pool local cache

[Suanming Mou]

For object which wants efficient index allocate and free, local
cache will be very helpful.

Two level cache is added, one as local and another as global.
The global cache is able to save all the allocated index. That
means all the allocated index will not be freed. Once the local
cache is full, the extra index will be flushed to the global
cache. Once local cache is empty, first try to fetch more index
from global, if global is still empty, allocate new trunk and
more index.

This commit adds a new local ring cache mechanism for indexed
pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 253 +++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_utils.h |  55 +++++++-
 2 files changed, 306 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index bf2b2ebc72..900de2a831 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -273,6 +273,22 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (cfg->per_core_cache) {
+		char cache_name[64] = { 0 };
+
+		for (i = 0; i < MLX5_IPOOL_MAX_CORES; i++) {
+			snprintf(cache_name, RTE_DIM(cache_name),
+				 "Ipool_cache-%p-%u", (void *)pool, i);
+			pool->l_idx_c[i] = rte_ring_create(cache_name,
+				cfg->per_core_cache, SOCKET_ID_ANY,
+				RING_F_SC_DEQ | RING_F_SP_ENQ);
+			if (!pool->l_idx_c[i]) {
+				printf("Ipool allocate ring failed:%d\n", i);
+				mlx5_free(pool);
+				return NULL;
+			}
+		}
+	}
 	return pool;
 }
 
@@ -355,6 +371,232 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static struct mlx5_indexed_trunk *
+mlx5_ipool_grow_cache(struct mlx5_indexed_pool *pool)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *pi;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx;
+	int n_grow;
+	int cidx = 0;
+	char cache_name[64] = { 0 };
+
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return NULL;
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (cidx == -1 || cidx > (MLX5_IPOOL_MAX_CORES - 1))
+		cidx = 0;
+	trunk_idx = __atomic_fetch_add(&pool->n_trunk_valid, 1,
+				__ATOMIC_ACQUIRE);
+	/* Have enough space in trunk array. Allocate the trunk directly. */
+	if (trunk_idx < __atomic_load_n(&pool->n_trunk, __ATOMIC_ACQUIRE))
+		goto allocate_trunk;
+	mlx5_ipool_lock(pool);
+	/* Double check if trunks array has been resized. */
+	if (trunk_idx < __atomic_load_n(&pool->n_trunk, __ATOMIC_ACQUIRE)) {
+		mlx5_ipool_unlock(pool);
+		goto allocate_trunk;
+	}
+	n_grow = trunk_idx ? pool->n_trunk :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk + n_grow);
+	/* Resize the trunk array. */
+	p = pool->cfg.malloc(MLX5_MEM_ZERO, ((trunk_idx + n_grow) *
+			     sizeof(struct mlx5_indexed_trunk *)) + sizeof(*p),
+			     RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!p) {
+		mlx5_ipool_unlock(pool);
+		return NULL;
+	}
+	p->trunks = (struct mlx5_indexed_trunk **)(p + 1);
+	if (pool->trunks_g)
+		memcpy(p->trunks, pool->trunks_g->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+	memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+	       n_grow * sizeof(void *));
+	/* Resize the global index cache ring. */
+	pi = pool->cfg.malloc(MLX5_MEM_ZERO, sizeof(*pi), 0, rte_socket_id());
+	if (!pi) {
+		mlx5_free(p);
+		mlx5_ipool_unlock(pool);
+		return NULL;
+	}
+	snprintf(cache_name, RTE_DIM(cache_name), "Idxc-%p-%u",
+		 (void *)pool, trunk_idx);
+	pi->ring = rte_ring_create(cache_name, rte_align32pow2(cur_max_idx),
+		SOCKET_ID_ANY, RING_F_SC_DEQ | RING_F_SP_ENQ);
+	if (!pi->ring) {
+		mlx5_free(p);
+		mlx5_free(pi);
+		mlx5_ipool_unlock(pool);
+		return NULL;
+	}
+	p->ref_cnt = 1;
+	pool->trunks_g = p;
+	pi->ref_cnt = 1;
+	pool->idx_g = pi;
+	/* Check if trunks array is not used any more. */
+	if (pool->trunks_c[cidx] && (!(--pool->trunks_c[cidx]->ref_cnt)))
+		mlx5_free(pool->trunks_c[cidx]);
+	/* Check if index cache is not used any more. */
+	if (pool->idx_c[cidx] &&
+	    (!(--pool->idx_c[cidx]->ref_cnt))) {
+		rte_ring_free(pool->idx_c[cidx]->ring);
+		mlx5_free(pool->idx_c[cidx]);
+	}
+	pool->trunks_c[cidx] = p;
+	pool->idx_c[cidx] = pi;
+	__atomic_fetch_add(&pool->n_trunk, n_grow, __ATOMIC_ACQUIRE);
+	mlx5_ipool_unlock(pool);
+allocate_trunk:
+	/* Initialize the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!trunk)
+		return NULL;
+	pool->trunks_g->trunks[trunk_idx] = trunk;
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+#ifdef POOL_DEBUG
+	pool->trunk_new++;
+#endif
+	return trunk;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx = 0;
+
+	if (!idx)
+		return NULL;
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (cidx == -1)
+		cidx = 0;
+	if (pool->trunks_c[cidx] != pool->trunks_g) {
+		mlx5_ipool_lock(pool);
+		/* Rlease the old ring if we are the last thread cache it. */
+		if (pool->trunks_c[cidx] &&
+		    !(--pool->trunks_c[cidx]->ref_cnt))
+			mlx5_free(pool->trunks_c[cidx]);
+		pool->trunks_c[cidx] = pool->trunks_g;
+		pool->trunks_c[cidx]->ref_cnt++;
+		mlx5_ipool_unlock(pool);
+	}
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = pool->trunks_c[cidx]->trunks[trunk_idx];
+	if (!trunk)
+		return NULL;
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	uint32_t i, ts_idx, fetch_size;
+	int cidx = 0;
+	union mlx5_indexed_qd qd;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (cidx == -1)
+		cidx = 0;
+	/* Try local cache firstly. */
+	if (!rte_ring_dequeue(pool->l_idx_c[cidx], &qd.ptr)) {
+		*idx = qd.idx;
+		return mlx5_ipool_get_cache(pool, qd.idx);
+	}
+	if (pool->idx_g) {
+		/*
+		 * Try fetch from the global cache. Check if global cache ring
+		 * updated first.
+		 */
+		if (pool->idx_c[cidx] != pool->idx_g) {
+			mlx5_ipool_lock(pool);
+			/* Rlease the old ring as last user. */
+			if (pool->idx_c[cidx] &&
+			    !(--pool->idx_c[cidx]->ref_cnt)) {
+				rte_ring_free(pool->idx_c[cidx]->ring);
+				pool->cfg.free(pool->idx_c[cidx]);
+			}
+			pool->idx_c[cidx] = pool->idx_g;
+			pool->idx_c[cidx]->ref_cnt++;
+			mlx5_ipool_unlock(pool);
+		}
+		fetch_size = pool->cfg.trunk_size;
+		while (!rte_ring_dequeue(pool->idx_g->ring, &qd.ptr)) {
+			if (unlikely(!(--fetch_size))) {
+				*idx = qd.idx;
+				return mlx5_ipool_get_cache(pool, qd.idx);
+			}
+			rte_ring_enqueue(pool->l_idx_c[cidx], qd.ptr);
+		}
+	}
+	/* Not enough idx in global cache. Keep fetching from new trunk. */
+	trunk = mlx5_ipool_grow_cache(pool);
+	if (!trunk)
+		return NULL;
+	/* Get trunk start index. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk->idx);
+	/* Enqueue trunk_size - 1 to local cache ring. */
+	for (i = 0; i < trunk->free - 1; i++) {
+		qd.idx = ts_idx + i + 1;
+		rte_ring_enqueue(pool->l_idx_c[cidx], qd.ptr);
+	}
+	/* Return trunk's final entry. */
+	*idx = ts_idx + i + 1;
+	return &trunk->data[i * pool->cfg.size];
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	uint32_t i, reclaim_num = 0;
+	union mlx5_indexed_qd qd;
+
+	if (!idx)
+		return;
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (cidx == -1)
+		cidx = 0;
+	qd.idx = idx;
+	/* Try to enqueue to local index cache. */
+	if (!rte_ring_enqueue(pool->l_idx_c[cidx], qd.ptr))
+		return;
+	/* Update the global index cache ring if needed. */
+	if (pool->idx_c[cidx] != pool->idx_g) {
+		mlx5_ipool_lock(pool);
+		/* Rlease the old ring if we are the last thread cache it. */
+		if (!(--pool->idx_c[cidx]->ref_cnt))
+			rte_ring_free(pool->idx_c[cidx]->ring);
+		pool->idx_c[cidx] = pool->idx_g;
+		pool->idx_c[cidx]->ref_cnt++;
+		mlx5_ipool_unlock(pool);
+	}
+	reclaim_num = pool->cfg.per_core_cache >> 4;
+	/* Local index cache full, try with global index cache. */
+	rte_ring_enqueue(pool->idx_c[cidx]->ring, qd.ptr);
+	/* Dequeue the index from local cache to global. */
+	for (i = 0; i < reclaim_num; i++) {
+		/* No need to check the return value. */
+		rte_ring_dequeue(pool->l_idx_c[cidx], &qd.ptr);
+		rte_ring_enqueue(pool->idx_c[cidx]->ring, qd.ptr);
+	}
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +605,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +676,8 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_free_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +743,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -524,7 +772,10 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
+	if (pool->cfg.per_core_cache)
+		trunks = pool->trunks_g->trunks;
+	else
+		trunks = pool->trunks;
 	for (i = 0; i < pool->n_trunk; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 15870e14c2..4fe82d4a5f 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -181,10 +181,17 @@ typedef int32_t (*mlx5_l3t_alloc_callback_fn)(void *ctx,
 #define TRUNK_MAX_IDX ((1 << TRUNK_IDX_BITS) - 1)
 #define TRUNK_INVALID TRUNK_MAX_IDX
 #define MLX5_IPOOL_DEFAULT_TRUNK_SIZE (1 << (28 - TRUNK_IDX_BITS))
+#define MLX5_IPOOL_MAX_CORES (1 << 4)
 #ifdef RTE_LIBRTE_MLX5_DEBUG
 #define POOL_DEBUG 1
 #endif
 
+union mlx5_indexed_qd {
+	RTE_STD_C11
+	void *ptr;
+	uint32_t idx;
+};
+
 struct mlx5_indexed_pool_config {
 	uint32_t size; /* Pool entry size. */
 	uint32_t trunk_size:22;
@@ -209,6 +216,11 @@ struct mlx5_indexed_pool_config {
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
 	uint32_t max_idx; /* The maximum index can be allocated. */
+	uint32_t per_core_cache;
+	/*
+	 * Cache entry number per core for performance. Should not be
+	 * set with release_mem_en.
+	 */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
@@ -217,6 +229,7 @@ struct mlx5_indexed_pool_config {
 };
 
 struct mlx5_indexed_trunk {
+	rte_spinlock_t lock; /* Trunk lock for multiple thread usage. */
 	uint32_t idx; /* Trunk id. */
 	uint32_t prev; /* Previous free trunk in free list. */
 	uint32_t next; /* Next free trunk in free list. */
@@ -225,13 +238,29 @@ struct mlx5_indexed_trunk {
 	uint8_t data[] __rte_cache_aligned; /* Entry data start. */
 };
 
+struct mlx5_indexed_cache {
+	union {
+		struct rte_ring *ring;
+		struct mlx5_indexed_trunk **trunks;
+	};
+	uint32_t ref_cnt;
+	uint32_t res;
+};
+
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t lock; /* Pool lock for multiple thread usage. */
 	uint32_t n_trunk_valid; /* Trunks allocated. */
 	uint32_t n_trunk; /* Trunk pointer array size. */
 	/* Dim of trunk pointer array. */
-	struct mlx5_indexed_trunk **trunks;
+	union {
+		struct mlx5_indexed_trunk **trunks;
+		struct mlx5_indexed_cache *trunks_g;
+	};
+	struct mlx5_indexed_cache *trunks_c[MLX5_IPOOL_MAX_CORES];
+	struct mlx5_indexed_cache *idx_g;
+	struct mlx5_indexed_cache *idx_c[MLX5_IPOOL_MAX_CORES];
+	struct rte_ring *l_idx_c[MLX5_IPOOL_MAX_CORES];
 	uint32_t free_list; /* Index to first free trunk. */
 #ifdef POOL_DEBUG
 	uint32_t n_entry;
@@ -542,6 +571,30 @@ int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool);
  */
 void mlx5_ipool_dump(struct mlx5_indexed_pool *pool);
 
+/**
+ * This function flushes all the cache index back to pool trunk.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ *
+ */
+
+void mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool);
+
+/**
+ * This function gets the available entry from pos.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ * @param pos
+ *   Pointer to the index position start from.
+ *
+ * @return
+ *  - Pointer to the next available entry.
+ *
+ */
+void *mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos);
+
 /**
  * This function allocates new empty Three-level table.
  *
-- 
2.25.1

[dpdk-dev] [PATCH 3/4] net/mlx5: add index pool cache flush

[Suanming Mou]

This commit adds the per-core cache indexed pool cache flush function.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 70 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h | 13 +++++++
 2 files changed, 83 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 900de2a831..ee6c2e293e 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -452,6 +452,21 @@ mlx5_ipool_grow_cache(struct mlx5_indexed_pool *pool)
 	pool->idx_c[cidx] = pi;
 	__atomic_fetch_add(&pool->n_trunk, n_grow, __ATOMIC_ACQUIRE);
 	mlx5_ipool_unlock(pool);
+	/* Pre-allocate the bitmap. */
+	if (pool->ibmp)
+		pool->cfg.free(pool->ibmp);
+	data_size = sizeof(*pool->ibmp);
+	data_size += rte_bitmap_get_memory_footprint(cur_max_idx);
+	/* rte_bitmap requires memory cacheline aligned. */
+	pool->ibmp = pool->cfg.malloc(MLX5_MEM_ZERO, data_size,
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->ibmp)
+		return NULL;
+	pool->ibmp->num = cur_max_idx;
+	pool->ibmp->mem_size = data_size - sizeof(*pool->ibmp);
+	pool->ibmp->mem = (void *)(pool->ibmp + 1);
+	pool->ibmp->bmp = rte_bitmap_init(pool->ibmp->num,
+			pool->ibmp->mem, pool->ibmp->mem_size);
 allocate_trunk:
 	/* Initialize the new trunk. */
 	trunk_size = sizeof(*trunk);
@@ -787,6 +802,61 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i;
+	struct rte_ring *ring_c;
+	char cache_name[64];
+	union mlx5_indexed_qd qd;
+	uint32_t bmp_num, mem_size;
+	uint32_t num = 0;
+
+	/* Create a new ring to save all cached index. */
+	snprintf(cache_name, RTE_DIM(cache_name), "Ip_%p",
+		 (void *)pool->idx_g->ring);
+	ring_c = rte_ring_create(cache_name, pool->ibmp->num,
+			SOCKET_ID_ANY, RING_F_SC_DEQ | RING_F_SP_ENQ);
+	if (!ring_c)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->ibmp->bmp = rte_bitmap_init_with_all_set(bmp_num,
+			pool->ibmp->mem, mem_size);
+	/* Flush core cache. */
+	for (i = 0; i < MLX5_IPOOL_MAX_CORES; i++) {
+		while (!rte_ring_dequeue(pool->l_idx_c[i], &qd.ptr)) {
+			rte_bitmap_clear(pool->ibmp->bmp, (qd.idx - 1));
+			rte_ring_enqueue(ring_c, qd.ptr);
+			num++;
+		}
+	}
+	/* Flush global cache. */
+	while (!rte_ring_dequeue(pool->idx_g->ring, &qd.ptr)) {
+		rte_bitmap_clear(pool->ibmp->bmp, (qd.idx - 1));
+		rte_ring_enqueue(ring_c, qd.ptr);
+		num++;
+	}
+	rte_ring_free(pool->idx_g->ring);
+	pool->idx_g->ring = ring_c;
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	if (!pool->ibmp || !rte_bitmap_scan(pool->ibmp->bmp, &iidx, &slab))
+		return NULL;
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(pool->ibmp->bmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get(pool, iidx);
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 4fe82d4a5f..03d5164485 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -247,6 +247,13 @@ struct mlx5_indexed_cache {
 	uint32_t res;
 };
 
+struct mlx5_indexed_bmp {
+	struct rte_bitmap *bmp;
+	void *mem;
+	uint32_t mem_size;
+	uint32_t num;
+} __rte_cache_aligned;
+
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t lock; /* Pool lock for multiple thread usage. */
@@ -261,6 +268,7 @@ struct mlx5_indexed_pool {
 	struct mlx5_indexed_cache *idx_g;
 	struct mlx5_indexed_cache *idx_c[MLX5_IPOOL_MAX_CORES];
 	struct rte_ring *l_idx_c[MLX5_IPOOL_MAX_CORES];
+	struct mlx5_indexed_bmp *ibmp;
 	uint32_t free_list; /* Index to first free trunk. */
 #ifdef POOL_DEBUG
 	uint32_t n_entry;
@@ -861,4 +869,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry) \
+	for ((idx) = 1, mlx5_ipool_flush_cache((ipool)), \
+	     entry = mlx5_ipool_get_next((ipool), &idx); \
+	     (entry); idx++, entry = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
-- 
2.25.1

[dpdk-dev] [PATCH 4/4] net/mlx5: replace flow list with index pool

[Suanming Mou]

The flow list is used to save the create flows and to be used only
when port closes all the flows need to be flushed.

This commit creates a new function to get the available entries from
the index pool. Then flows can be flushed from index pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c |  11 +++
 drivers/net/mlx5/mlx5.c          |   3 +-
 drivers/net/mlx5/mlx5.h          |   4 +-
 drivers/net/mlx5/mlx5_flow.c     | 111 +++++++++++--------------------
 drivers/net/mlx5/mlx5_flow.h     |   1 +
 drivers/net/mlx5/mlx5_flow_dv.c  |   5 ++
 drivers/net/mlx5/mlx5_trigger.c  |   8 +--
 7 files changed, 61 insertions(+), 82 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 534a56a555..c56be18d24 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1608,6 +1608,17 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	struct mlx5_indexed_pool_config icfg = {
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 4096,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 23,
+		.type = "rte_flow_ipool",
+	};
+	priv->flows = mlx5_ipool_create(&icfg);
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..b0f9f1a45e 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -323,6 +323,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_shift = 2,
 		.need_lock = 1,
 		.release_mem_en = 1,
+		.per_core_cache = 1 << 23,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -1528,7 +1529,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, false, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 32b2817bf2..f6b72cd352 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1344,7 +1344,7 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows; /* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1596,7 +1596,7 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, bool control, bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index dbeca571b6..8f9eea2c00 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3068,31 +3068,6 @@ mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
 					 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
 }
 
-/**
- * Release resource related QUEUE/RSS action split.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Flow to release id's from.
- */
-static void
-flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
-			     struct rte_flow *flow)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t handle_idx;
-	struct mlx5_flow_handle *dev_handle;
-
-	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
-		       handle_idx, dev_handle, next)
-		if (dev_handle->split_flow_id &&
-		    !dev_handle->is_meter_flow_id)
-			mlx5_ipool_free(priv->sh->ipool
-					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
-					dev_handle->split_flow_id);
-}
-
 static int
 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
 		   const struct rte_flow_attr *attr __rte_unused,
@@ -3388,7 +3363,6 @@ flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 	const struct mlx5_flow_driver_ops *fops;
 	enum mlx5_flow_drv_type type = flow->drv_type;
 
-	flow_mreg_split_qrss_release(dev, flow);
 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
 	fops = flow_get_drv_ops(type);
 	fops->destroy(dev, flow);
@@ -3971,7 +3945,7 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
 
 /* Declare flow create/destroy prototype in advance. */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, bool control,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action actions[],
@@ -4100,7 +4074,7 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	 * be applied, removed, deleted in ardbitrary order
 	 * by list traversing.
 	 */
-	mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
+	mcp_res->rix_flow = flow_list_create(dev, false, &attr, items,
 					 actions, false, error);
 	if (!mcp_res->rix_flow) {
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
@@ -6062,7 +6036,7 @@ flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
  *   A flow index on success, 0 otherwise and rte_errno is set.
  */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, bool control,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action original_actions[],
@@ -6130,7 +6104,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 				external, hairpin_flow, error);
 	if (ret < 0)
 		goto error_before_hairpin_split;
-	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
+	flow = mlx5_ipool_zmalloc(priv->flows, &idx);
 	if (!flow) {
 		rte_errno = ENOMEM;
 		goto error_before_hairpin_split;
@@ -6256,12 +6230,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 		if (ret < 0)
 			goto error;
 	}
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
-			     flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
+	flow->control = control;
 	flow_rxq_flags_set(dev, flow);
 	rte_free(translated_actions);
 	tunnel = flow_tunnel_from_rule(wks->flows);
@@ -6283,7 +6252,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_ipool_get
 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
 			rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
+	mlx5_ipool_free(priv->flows, idx);
 	rte_errno = ret; /* Restore rte_errno. */
 	ret = rte_errno;
 	rte_errno = ret;
@@ -6335,10 +6304,9 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
 			.type = RTE_FLOW_ACTION_TYPE_END,
 		},
 	};
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow_error error;
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
+	return (void *)(uintptr_t)flow_list_create(dev, true,
 						   &attr, &pattern,
 						   actions, false, &error);
 }
@@ -6390,8 +6358,6 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
 	/*
 	 * If the device is not started yet, it is not allowed to created a
 	 * flow from application. PMD default flows and traffic control flows
@@ -6407,7 +6373,7 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		return NULL;
 	}
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
+	return (void *)(uintptr_t)flow_list_create(dev, false,
 				  attr, items, actions, true, error);
 }
 
@@ -6429,8 +6395,7 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 		  uint32_t flow_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW], flow_idx);
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows, flow_idx);
 
 	if (!flow)
 		return;
@@ -6443,7 +6408,7 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 	flow_drv_destroy(dev, flow);
 	if (list) {
 		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
+		ILIST_REMOVE(priv->flows, list,
 			     flow_idx, flow, next);
 		rte_spinlock_unlock(&priv->flow_list_lock);
 	}
@@ -6456,7 +6421,7 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
 	flow_mreg_del_copy_action(dev, flow);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
+	mlx5_ipool_free(priv->flows, flow_idx);
 }
 
 /**
@@ -6464,19 +6429,23 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list.
+ * @param control
+ *   Flow type to be flushed.
  * @param active
  *   If flushing is called avtively.
  */
 void
-mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
+mlx5_flow_list_flush(struct rte_eth_dev *dev, bool control, bool active)
 {
-	uint32_t num_flushed = 0;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t num_flushed = 0, fidx = 1;
+	struct rte_flow *flow;
 
-	while (*list) {
-		flow_list_destroy(dev, list, *list);
-		num_flushed++;
+	MLX5_IPOOL_FOREACH(priv->flows, fidx, flow) {
+		if (flow->control == control) {
+			flow_list_destroy(dev, NULL, fidx);
+			num_flushed++;
+		}
 	}
 	if (active) {
 		DRV_LOG(INFO, "port %u: %u flows flushed before stopping",
@@ -6647,18 +6616,20 @@ mlx5_flow_pop_thread_workspace(void)
  * @return the number of flows not released.
  */
 int
-mlx5_flow_verify(struct rte_eth_dev *dev)
+mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
-	uint32_t idx;
+	uint32_t idx = 0;
 	int ret = 0;
 
-	ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
-		      flow, next) {
+	flow = mlx5_ipool_get_next(priv->flows, &idx);
+	while (flow) {
 		DRV_LOG(DEBUG, "port %u flow %p still referenced",
 			dev->data->port_id, (void *)flow);
-		++ret;
+		idx++;
+		ret++;
+		flow = mlx5_ipool_get_next(priv->flows, &idx);
 	}
 	return ret;
 }
@@ -6678,7 +6649,6 @@ int
 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 			    uint32_t queue)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.egress = 1,
 		.priority = 0,
@@ -6711,7 +6681,7 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 	actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
 	actions[0].conf = &jump;
 	actions[1].type = RTE_FLOW_ACTION_TYPE_END;
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
+	flow_idx = flow_list_create(dev, true,
 				&attr, items, actions, false, &error);
 	if (!flow_idx) {
 		DRV_LOG(DEBUG,
@@ -6801,7 +6771,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 		action_rss.types = 0;
 	for (i = 0; i != priv->reta_idx_n; ++i)
 		queue[i] = (*priv->reta_idx)[i];
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
+	flow_idx = flow_list_create(dev, true,
 				&attr, items, actions, false, &error);
 	if (!flow_idx)
 		return -rte_errno;
@@ -6843,7 +6813,6 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev,
 int
 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	/*
 	 * The LACP matching is done by only using ether type since using
 	 * a multicast dst mac causes kernel to give low priority to this flow.
@@ -6877,7 +6846,7 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 		},
 	};
 	struct rte_flow_error error;
-	uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
+	uint32_t flow_idx = flow_list_create(dev, true,
 				&attr, items, actions, false, &error);
 
 	if (!flow_idx)
@@ -6896,9 +6865,7 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
 		  struct rte_flow *flow,
 		  struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
+	flow_list_destroy(dev, NULL, (uintptr_t)(void *)flow);
 	return 0;
 }
 
@@ -6912,9 +6879,7 @@ int
 mlx5_flow_flush(struct rte_eth_dev *dev,
 		struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows, false);
+	mlx5_flow_list_flush(dev, false, false);
 	return 0;
 }
 
@@ -6965,8 +6930,7 @@ flow_drv_query(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct mlx5_flow_driver_ops *fops;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW],
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows,
 					       flow_idx);
 	enum mlx5_flow_drv_type ftype;
 
@@ -7832,10 +7796,9 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		if (!config->dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
-		flow_idx = flow_list_create(dev, NULL, &attr, items,
+		flow_idx = flow_list_create(dev, false, &attr, items,
 					    actions, false, &error);
-		flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
-				      flow_idx);
+		flow = mlx5_ipool_get(priv->flows, flow_idx);
 		if (!flow)
 			continue;
 		config->flow_mreg_c[n++] = idx;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2f2aa962f9..cb0b97a2d4 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1000,6 +1000,7 @@ struct rte_flow {
 	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t control:1;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index c50649a107..d039794c9a 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13830,6 +13830,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ae7fcca229..48cf780786 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, false, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, true, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, true, false);
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v2 00/22] net/mlx5: insertion rate optimization

[Suanming Mou]

This patch series optimize the flow insertion rate with adding
local cache to index pool and list.

For object which wants efficient index allocate and free, local
cache will be very helpful.

For index pool, two level cache is added, one as local and another
as global. The global cache is able to save all the allocated
index. That means all the allocated index will not be freed. Once
the local cache is full, the extra index will be flushed to the
global cache. Once local cache is empty, first try to fetch more
index from global, if global is still empty, allocate new trunk
and more index.

For list, sub local core list is introduced. The allocated objects
will be added and released only from local list without any locks.
Only the objects need to be shared will be synced from global list.

---

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
  net/mlx5: optimize modify header action memory
  net/mlx5: remove cache term from the list utility
  net/mlx5: add per lcore cache to the list utility
  net/mlx5: minimize list critical sections
  net/mlx5: manage list cache entries release
  net/mlx5: relax the list utility atomic operations
  net/mlx5: allocate list memory by the create API
  common/mlx5: add per-lcore cache to hash list utility
  net/mlx5: move modify header allocator to ipool

Suanming Mou (13):
  net/mlx5: allow limiting the index pool maximum index
  net/mlx5: add indexed pool local cache
  net/mlx5: add index pool foreach define
  net/mlx5: replace flow list with index pool
  net/mlx5: adjust the hash bucket size
  common/mlx5: allocate cache list memory individually
  net/mlx5: enable index pool per-core cache
  net/mlx5: optimize hash list table allocate on demand
  common/mlx5: optimize cache list object memory
  net/mlx5: change memory release configuration
  net/mlx5: support index pool none local core operations
  net/mlx5: support list none local core operations
  net/mlx5: optimize Rx queue match

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    6 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  283 ++++--
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  305 +++---
 drivers/net/mlx5/mlx5_flow.h            |  210 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1203 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  617 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 20 files changed, 2518 insertions(+), 1411 deletions(-)

-- 
2.25.1

[dpdk-dev] [PATCH v2 01/22] net/mlx5: allow limiting the index pool maximum index

[Suanming Mou]

Some ipool instances in the driver are used as ID\index allocator and
added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value.
The ipool will ensure that no index bigger than the maximum value is
provided.

Use this configuration in ID allocator cases instead of the current
logics. This patch add the maximum ID configurable for the index pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 18fe23e4fb..bf2b2ebc72 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index b54517c6df..15870e14c2 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
-- 
2.25.1

[dpdk-dev] [PATCH v2 02/22] net/mlx5: add indexed pool local cache

[Suanming Mou]

For object which wants efficient index allocate and free, local
cache will be very helpful.

Two level cache is introduced to allocate and free the index more
efficient. One as local and the other as global. The global cache
is able to save all the allocated index. That means all the allocated
index will not be freed. Once the local cache is full, the extra
index will be flushed to the global cache. Once local cache is empty,
first try to fetch more index from global, if global is still empty,
allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index bf2b2ebc72..215024632d 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 15870e14c2..0469062695 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -209,6 +209,11 @@ struct mlx5_indexed_pool_config {
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
 	uint32_t max_idx; /* The maximum index can be allocated. */
+	uint32_t per_core_cache;
+	/*
+	 * Cache entry number per core for performance. Should not be
+	 * set with release_mem_en.
+	 */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
@@ -225,14 +230,39 @@ struct mlx5_indexed_trunk {
 	uint8_t data[] __rte_cache_aligned; /* Entry data start. */
 };
 
+struct mlx5_indexed_cache {
+	struct mlx5_indexed_trunk **trunks;
+	volatile uint32_t n_trunk_valid; /* Trunks allocated. */
+	uint32_t n_trunk; /* Trunk pointer array size. */
+	uint32_t ref_cnt;
+	uint32_t len;
+	uint32_t idx[];
+};
+
+struct mlx5_ipool_per_lcore {
+	struct mlx5_indexed_cache *lc;
+	uint32_t len; /**< Current cache count. */
+	uint32_t idx[]; /**< Cache objects. */
+};
+
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
-	rte_spinlock_t lock; /* Pool lock for multiple thread usage. */
-	uint32_t n_trunk_valid; /* Trunks allocated. */
-	uint32_t n_trunk; /* Trunk pointer array size. */
+	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
 	/* Dim of trunk pointer array. */
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t free_list; /* Index to first free trunk. */
+	union {
+		struct {
+			uint32_t n_trunk_valid; /* Trunks allocated. */
+			uint32_t n_trunk; /* Trunk pointer array size. */
+			struct mlx5_indexed_trunk **trunks;
+			uint32_t free_list; /* Index to first free trunk. */
+		};
+		struct {
+			struct mlx5_indexed_cache *gc;
+			/* Global cache. */
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			/* Local cache. */
+		};
+	};
 #ifdef POOL_DEBUG
 	uint32_t n_entry;
 	uint32_t trunk_new;
@@ -542,6 +572,30 @@ int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool);
  */
 void mlx5_ipool_dump(struct mlx5_indexed_pool *pool);
 
+/**
+ * This function flushes all the cache index back to pool trunk.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ *
+ */
+
+void mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool);
+
+/**
+ * This function gets the available entry from pos.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ * @param pos
+ *   Pointer to the index position start from.
+ *
+ * @return
+ *  - Pointer to the next available entry.
+ *
+ */
+void *mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos);
+
 /**
  * This function allocates new empty Three-level table.
  *
-- 
2.25.1

[dpdk-dev] [PATCH v2 03/22] net/mlx5: add index pool foreach define

[Suanming Mou]

In some cases, application may want to know all the allocated
index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach
operation.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 86 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 ++++
 2 files changed, 94 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 215024632d..0ed279e162 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -839,6 +839,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 0469062695..737dd7052d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
-- 
2.25.1

[dpdk-dev] [PATCH v2 04/22] net/mlx5: replace flow list with index pool

[Suanming Mou]

The flow list is used to save the create flows and to be used only
when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to
flush all the allocated flows.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 +++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 149 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 126 insertions(+), 110 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 92b3009786..31cc8d9eb8 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -823,6 +861,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1566,7 +1605,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1600,6 +1638,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..fcfc3dcdca 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -322,7 +322,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -792,8 +793,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1528,7 +1531,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 32b2817bf2..5fa5d3cb99 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1344,7 +1352,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1596,7 +1605,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c5d4a95a8f..20ce0ed424 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3095,31 +3095,6 @@ mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
 					 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
 }
 
-/**
- * Release resource related QUEUE/RSS action split.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Flow to release id's from.
- */
-static void
-flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
-			     struct rte_flow *flow)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t handle_idx;
-	struct mlx5_flow_handle *dev_handle;
-
-	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
-		       handle_idx, dev_handle, next)
-		if (dev_handle->split_flow_id &&
-		    !dev_handle->is_meter_flow_id)
-			mlx5_ipool_free(priv->sh->ipool
-					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
-					dev_handle->split_flow_id);
-}
-
 static int
 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
 		   const struct rte_flow_attr *attr __rte_unused,
@@ -3415,7 +3390,6 @@ flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 	const struct mlx5_flow_driver_ops *fops;
 	enum mlx5_flow_drv_type type = flow->drv_type;
 
-	flow_mreg_split_qrss_release(dev, flow);
 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
 	fops = flow_get_drv_ops(type);
 	fops->destroy(dev, flow);
@@ -3998,14 +3972,14 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
 
 /* Declare flow create/destroy prototype in advance. */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action actions[],
 		 bool external, struct rte_flow_error *error);
 
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
@@ -4127,8 +4101,8 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	 * be applied, removed, deleted in ardbitrary order
 	 * by list traversing.
 	 */
-	mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
-					 actions, false, error);
+	mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
+					&attr, items, actions, false, error);
 	if (!mcp_res->rix_flow) {
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
 		return NULL;
@@ -4190,7 +4164,7 @@ flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
-	flow_list_destroy(dev, NULL, mcp_res->rix_flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
 }
 
@@ -6093,7 +6067,7 @@ flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
  *   A flow index on success, 0 otherwise and rte_errno is set.
  */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action original_actions[],
@@ -6161,7 +6135,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 				external, hairpin_flow, error);
 	if (ret < 0)
 		goto error_before_hairpin_split;
-	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
+	flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
 	if (!flow) {
 		rte_errno = ENOMEM;
 		goto error_before_hairpin_split;
@@ -6291,12 +6265,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 		if (ret < 0)
 			goto error;
 	}
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
-			     flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
+	flow->type = type;
 	flow_rxq_flags_set(dev, flow);
 	rte_free(translated_actions);
 	tunnel = flow_tunnel_from_rule(wks->flows);
@@ -6318,7 +6287,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_ipool_get
 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
 			rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
+	mlx5_ipool_free(priv->flows[type], idx);
 	rte_errno = ret; /* Restore rte_errno. */
 	ret = rte_errno;
 	rte_errno = ret;
@@ -6370,10 +6339,9 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
 			.type = RTE_FLOW_ACTION_TYPE_END,
 		},
 	};
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow_error error;
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
 						   &attr, &pattern,
 						   actions, false, &error);
 }
@@ -6425,8 +6393,6 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
 	/*
 	 * If the device is not started yet, it is not allowed to created a
 	 * flow from application. PMD default flows and traffic control flows
@@ -6442,8 +6408,9 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		return NULL;
 	}
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
-				  attr, items, actions, true, error);
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
+						   attr, items, actions,
+						   true, error);
 }
 
 /**
@@ -6451,24 +6418,19 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list. If this parameter NULL,
- *   there is no flow removal from the list. Be noted that as
- *   flow is add to the indexed list, memory of the indexed
- *   list points to maybe changed as flow destroyed.
  * @param[in] flow_idx
  *   Index of flow to destroy.
  */
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW], flow_idx);
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
 
 	if (!flow)
 		return;
+	MLX5_ASSERT(flow->type == type);
 	/*
 	 * Update RX queue flags only if port is started, otherwise it is
 	 * already clean.
@@ -6476,12 +6438,6 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
 	flow_drv_destroy(dev, flow);
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
-			     flow_idx, flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
 	if (flow->tunnel) {
 		struct mlx5_flow_tunnel *tunnel;
 
@@ -6491,7 +6447,7 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
 	flow_mreg_del_copy_action(dev, flow);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
+	mlx5_ipool_free(priv->flows[type], flow_idx);
 }
 
 /**
@@ -6499,18 +6455,21 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list.
+ * @param type
+ *   Flow type to be flushed.
  * @param active
  *   If flushing is called avtively.
  */
 void
-mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
+mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+		     bool active)
 {
-	uint32_t num_flushed = 0;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t num_flushed = 0, fidx = 1;
+	struct rte_flow *flow;
 
-	while (*list) {
-		flow_list_destroy(dev, list, *list);
+	MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
+		flow_list_destroy(dev, type, fidx);
 		num_flushed++;
 	}
 	if (active) {
@@ -6682,18 +6641,19 @@ mlx5_flow_pop_thread_workspace(void)
  * @return the number of flows not released.
  */
 int
-mlx5_flow_verify(struct rte_eth_dev *dev)
+mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
-	uint32_t idx;
-	int ret = 0;
+	uint32_t idx = 0;
+	int ret = 0, i;
 
-	ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
-		      flow, next) {
-		DRV_LOG(DEBUG, "port %u flow %p still referenced",
-			dev->data->port_id, (void *)flow);
-		++ret;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
+			DRV_LOG(DEBUG, "port %u flow %p still referenced",
+				dev->data->port_id, (void *)flow);
+			ret++;
+		}
 	}
 	return ret;
 }
@@ -6713,7 +6673,6 @@ int
 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 			    uint32_t queue)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.egress = 1,
 		.priority = 0,
@@ -6746,8 +6705,8 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 	actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
 	actions[0].conf = &jump;
 	actions[1].type = RTE_FLOW_ACTION_TYPE_END;
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx) {
 		DRV_LOG(DEBUG,
 			"Failed to create ctrl flow: rte_errno(%d),"
@@ -6836,8 +6795,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 		action_rss.types = 0;
 	for (i = 0; i != priv->reta_idx_n; ++i)
 		queue[i] = (*priv->reta_idx)[i];
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx)
 		return -rte_errno;
 	return 0;
@@ -6878,7 +6837,6 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev,
 int
 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	/*
 	 * The LACP matching is done by only using ether type since using
 	 * a multicast dst mac causes kernel to give low priority to this flow.
@@ -6912,8 +6870,9 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 		},
 	};
 	struct rte_flow_error error;
-	uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+					&attr, items, actions,
+					false, &error);
 
 	if (!flow_idx)
 		return -rte_errno;
@@ -6931,9 +6890,8 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
 		  struct rte_flow *flow,
 		  struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+				(uintptr_t)(void *)flow);
 	return 0;
 }
 
@@ -6947,9 +6905,7 @@ int
 mlx5_flow_flush(struct rte_eth_dev *dev,
 		struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
 	return 0;
 }
 
@@ -7000,8 +6956,7 @@ flow_drv_query(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct mlx5_flow_driver_ops *fops;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW],
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 					       flow_idx);
 	enum mlx5_flow_drv_type ftype;
 
@@ -7867,14 +7822,14 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		if (!config->dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
-		flow_idx = flow_list_create(dev, NULL, &attr, items,
-					    actions, false, &error);
-		flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
+					items, actions, false, &error);
+		flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 				      flow_idx);
 		if (!flow)
 			continue;
 		config->flow_mreg_c[n++] = idx;
-		flow_list_destroy(dev, NULL, flow_idx);
+		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
 	}
 	for (; n < MLX5_MREG_C_NUM; ++n)
 		config->flow_mreg_c[n] = REG_NON;
@@ -7918,8 +7873,8 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
 					sh->rx_domain,
 					sh->tx_domain, file);
 	/* dump one */
-	flow = mlx5_ipool_get(priv->sh->ipool
-			[MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
+	flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
+			(uintptr_t)(void *)flow_idx);
 	if (!flow)
 		return -ENOENT;
 
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2f2aa962f9..d9b6acaafd 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -997,9 +997,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index c5d4b01e57..67f7243503 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13844,6 +13844,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ae7fcca229..7cb8920d6b 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 3fe3f55f49..7d15c998bb 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	TAILQ_INIT(&priv->flow_meter_profiles);
-- 
2.25.1

[dpdk-dev] [PATCH v2 05/22] net/mlx5: optimize modify header action memory

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Define the types of the modify header action fields to be with the
minimum size needed for the optional values range.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_glue.h b/drivers/common/mlx5/linux/mlx5_glue.h
index 840d8cf57f..a186ee577f 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.h
+++ b/drivers/common/mlx5/linux/mlx5_glue.h
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
diff --git a/drivers/net/mlx5/linux/mlx5_flow_os.h b/drivers/net/mlx5/linux/mlx5_flow_os.h
index cee685015b..1926d26410 100644
--- a/drivers/net/mlx5/linux/mlx5_flow_os.h
+++ b/drivers/net/mlx5/linux/mlx5_flow_os.h
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index d9b6acaafd..81c95e0beb 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -523,11 +523,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 67f7243503..784ec11dea 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5000,21 +5000,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5582,10 +5582,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
-- 
2.25.1

[dpdk-dev] [PATCH v2 06/22] net/mlx5: remove cache term from the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The internal mlx5 list tool is used mainly when the list objects need to
be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread
cache management.

To prevent confusing, remove the current "cache" term from the API's
names.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 558 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 454 insertions(+), 460 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 31cc8d9eb8..9aa57e38b7 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -500,8 +500,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1702,7 +1702,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1761,7 +1761,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index fcfc3dcdca..9aade013c5 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1611,7 +1611,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 5fa5d3cb99..803387b98e 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1221,7 +1221,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1369,7 +1369,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 81c95e0beb..4dec703366 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -467,7 +467,7 @@ struct mlx5_flow_dv_match_params {
 
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
-	struct mlx5_cache_entry entry; /**< Pointer to the next element. */
+	struct mlx5_list_entry entry; /**< Pointer to the next element. */
 	struct mlx5_flow_tbl_resource *tbl;
 	/**< Pointer to the table(group) the matcher associated with. */
 	void *matcher_object; /**< Pointer to DV matcher */
@@ -547,7 +547,7 @@ struct mlx5_flow_dv_jump_tbl_resource {
 
 /* Port ID resource structure. */
 struct mlx5_flow_dv_port_id_action_resource {
-	struct mlx5_cache_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Action object. */
 	uint32_t port_id; /**< Port ID value. */
 	uint32_t idx; /**< Indexed pool memory index. */
@@ -555,7 +555,7 @@ struct mlx5_flow_dv_port_id_action_resource {
 
 /* Push VLAN action resource structure */
 struct mlx5_flow_dv_push_vlan_action_resource {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* Cache entry. */
 	void *action; /**< Action object. */
 	uint8_t ft_type; /**< Flow table type, Rx, Tx or FDB. */
 	rte_be32_t vlan_tag; /**< VLAN tag value. */
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_cache_list matchers;
+	struct mlx5_list matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
@@ -631,7 +631,7 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	union {
 		void *verbs_action; /**< Verbs sample action object. */
 		void **sub_actions; /**< Sample sub-action array. */
@@ -653,7 +653,7 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
@@ -1619,43 +1619,45 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
 
-int flow_dv_matcher_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *ctx);
-struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_port_id_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_sample_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
-			      struct mlx5_cache_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
-				struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_dest_array_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry);
+int flow_dv_matcher_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *ctx);
+void flow_dv_matcher_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
+						 struct mlx5_list_entry *entry,
+						 void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 784ec11dea..ed0381329f 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3601,18 +3601,17 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 			     uint64_t key __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
-
-	cache_resource = container_of(entry,
-				      struct mlx5_flow_dv_encap_decap_resource,
-				      entry);
-	if (resource->reformat_type == cache_resource->reformat_type &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->flags == cache_resource->flags &&
-	    resource->size == cache_resource->size &&
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
+
+	resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
+				entry);
+	if (resource->reformat_type == ctx_resource->reformat_type &&
+	    resource->ft_type == ctx_resource->ft_type &&
+	    resource->flags == ctx_resource->flags &&
+	    resource->size == ctx_resource->size &&
 	    !memcmp((const void *)resource->buf,
-		    (const void *)cache_resource->buf,
+		    (const void *)ctx_resource->buf,
 		    resource->size))
 		return 0;
 	return -1;
@@ -3639,31 +3638,30 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 	uint32_t idx;
 	int ret;
 
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
 	/* Register new encap/decap resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-				       &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
-	cache_resource->idx = idx;
-	ret = mlx5_flow_os_create_flow_action_packet_reformat
-					(sh->ctx, domain, cache_resource,
-					 &cache_resource->action);
+	*resource = *ctx_resource;
+	resource->idx = idx;
+	ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
+							      resource,
+							     &resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3672,7 +3670,7 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 		return NULL;
 	}
 
-	return &cache_resource->entry;
+	return &resource->entry;
 }
 
 /**
@@ -3776,8 +3774,8 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
@@ -3787,30 +3785,30 @@ flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->port_id != res->port_id;
 }
 
-struct mlx5_cache_entry *
-flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_port_id_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate port_id action cache memory");
+				   "cannot allocate port_id action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
 							ref->port_id,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3818,8 +3816,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 				   "cannot create action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3827,8 +3825,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3840,30 +3838,30 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_port_id_action_resource_register
 			(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_port_id_action_resource *resource,
+			 struct mlx5_flow_dv_port_id_action_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->dv.port_id_action = cache;
-	dev_flow->handle->rix_port_id_action = cache->idx;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->dv.port_id_action = resource;
+	dev_flow->handle->rix_port_id_action = resource->idx;
 	return 0;
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
@@ -3873,28 +3871,28 @@ flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
-struct mlx5_cache_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	struct mlx5dv_dr_domain *domain;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate push_vlan action cache memory");
+				   "cannot allocate push_vlan action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
@@ -3902,7 +3900,7 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 	else
 		domain = sh->tx_domain;
 	ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3910,8 +3908,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 				   "cannot create push vlan action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3919,8 +3917,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
  *
  * @param [in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3932,25 +3930,25 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_push_vlan_action_resource_register
 		       (struct rte_eth_dev *dev,
-			struct mlx5_flow_dv_push_vlan_action_resource *resource,
+			struct mlx5_flow_dv_push_vlan_action_resource *ref,
 			struct mlx5_flow *dev_flow,
 			struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
+	resource = container_of(entry, typeof(*resource), entry);
 
-	dev_flow->handle->dvh.rix_push_vlan = cache->idx;
-	dev_flow->dv.push_vlan_res = cache;
+	dev_flow->handle->dvh.rix_push_vlan = resource->idx;
+	dev_flow->dv.push_vlan_res = resource;
 	return 0;
 }
 
@@ -9913,13 +9911,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 			return NULL;
 		}
 	}
-	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache",
+	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh,
-			     flow_dv_matcher_create_cb,
-			     flow_dv_matcher_match_cb,
-			     flow_dv_matcher_remove_cb);
+	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+			 flow_dv_matcher_create_cb,
+			 flow_dv_matcher_match_cb,
+			 flow_dv_matcher_remove_cb);
 	return &tbl_data->entry;
 }
 
@@ -10047,7 +10045,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_cache_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(&tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10075,8 +10073,8 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
@@ -10089,15 +10087,15 @@ flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
 		      (const void *)ref->mask.buf, ref->mask.size);
 }
 
-struct mlx5_cache_entry *
-flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_matcher_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5dv_flow_matcher_attr dv_attr = {
 		.type = IBV_FLOW_ATTR_NORMAL,
 		.match_mask = (void *)&ref->mask,
@@ -10106,29 +10104,30 @@ flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
 							    typeof(*tbl), tbl);
 	int ret;
 
-	cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY);
-	if (!cache) {
+	resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
+			       SOCKET_ID_ANY);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	dv_attr.match_criteria_enable =
-		flow_dv_matcher_enable(cache->mask.buf);
+		flow_dv_matcher_enable(resource->mask.buf);
 	dv_attr.priority = ref->priority;
 	if (tbl->is_egress)
 		dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
 	ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
-					       &cache->matcher_object);
+					       &resource->matcher_object);
 	if (ret) {
-		mlx5_free(cache);
+		mlx5_free(resource);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	return &cache->entry;
+	return &resource->entry;
 }
 
 /**
@@ -10157,8 +10156,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 			 uint32_t group_id,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5_flow_tbl_resource *tbl;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 	struct mlx5_flow_cb_ctx ctx = {
@@ -10178,15 +10177,15 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 					  "cannot allocate ref memory");
 	}
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->handle->dvh.matcher = cache;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.matcher = resource;
 	return 0;
 }
 
@@ -10254,15 +10253,15 @@ flow_dv_tag_resource_register
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_tag_resource *cache_resource;
+	struct mlx5_flow_dv_tag_resource *resource;
 	struct mlx5_hlist_entry *entry;
 
 	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
 	if (entry) {
-		cache_resource = container_of
-			(entry, struct mlx5_flow_dv_tag_resource, entry);
-		dev_flow->handle->dvh.rix_tag = cache_resource->idx;
-		dev_flow->dv.tag_resource = cache_resource;
+		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
+					entry);
+		dev_flow->handle->dvh.rix_tag = resource->idx;
+		dev_flow->dv.tag_resource = resource;
 		return 0;
 	}
 	return -rte_errno;
@@ -10589,68 +10588,69 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused,
-			struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-
-	if (resource->ratio == cache_resource->ratio &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->ft_id == cache_resource->ft_id &&
-	    resource->set_action == cache_resource->set_action &&
-	    !memcmp((void *)&resource->sample_act,
-		    (void *)&cache_resource->sample_act,
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+
+	if (ctx_resource->ratio == resource->ratio &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    ctx_resource->ft_id == resource->ft_id &&
+	    ctx_resource->set_action == resource->set_action &&
+	    !memcmp((void *)&ctx_resource->sample_act,
+		    (void *)&resource->sample_act,
 		    sizeof(struct mlx5_flow_sub_actions_list))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
 		flow_dv_sample_sub_actions_release(dev,
-						&resource->sample_idx);
+						   &ctx_resource->sample_idx);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	void **sample_dv_actions = resource->sub_actions;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	void **sample_dv_actions = ctx_resource->sub_actions;
+	struct mlx5_flow_dv_sample_resource *resource;
 	struct mlx5dv_dr_flow_sampler_attr sampler_attr;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	struct mlx5_flow_tbl_resource *tbl;
 	uint32_t idx = 0;
 	const uint32_t next_ft_step = 1;
-	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
+	uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
 	uint8_t is_egress = 0;
 	uint8_t is_transfer = 0;
 	struct rte_flow_error *error = ctx->error;
 
 	/* Register new sample resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	/* Create normal path table level */
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		is_transfer = 1;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
 		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
 					is_egress, is_transfer,
@@ -10663,8 +10663,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 					  "for sample");
 		goto error;
 	}
-	cache_resource->normal_path_tbl = tbl;
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
+	resource->normal_path_tbl = tbl;
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
 		if (!sh->default_miss_action) {
 			rte_flow_error_set(error, ENOMEM,
 						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10673,33 +10673,33 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 						"created");
 			goto error;
 		}
-		sample_dv_actions[resource->sample_act.actions_num++] =
+		sample_dv_actions[ctx_resource->sample_act.actions_num++] =
 						sh->default_miss_action;
 	}
 	/* Create a DR sample action */
-	sampler_attr.sample_ratio = cache_resource->ratio;
+	sampler_attr.sample_ratio = resource->ratio;
 	sampler_attr.default_next_table = tbl->obj;
-	sampler_attr.num_sample_actions = resource->sample_act.actions_num;
+	sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
 	sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
 							&sample_dv_actions[0];
-	sampler_attr.action = cache_resource->set_action;
+	sampler_attr.action = resource->set_action;
 	if (mlx5_os_flow_dr_create_flow_action_sampler
-			(&sampler_attr, &cache_resource->verbs_action)) {
+			(&sampler_attr, &resource->verbs_action)) {
 		rte_flow_error_set(error, ENOMEM,
 					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					NULL, "cannot create sample action");
 		goto error;
 	}
-	cache_resource->idx = idx;
-	cache_resource->dev = dev;
-	return &cache_resource->entry;
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
 error:
-	if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
 		flow_dv_sample_sub_actions_release(dev,
-						   &cache_resource->sample_idx);
-	if (cache_resource->normal_path_tbl)
+						   &resource->sample_idx);
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-				cache_resource->normal_path_tbl);
+				resource->normal_path_tbl);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
 	return NULL;
 
@@ -10710,8 +10710,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to sample resource.
+ * @param[in] ref
+ *   Pointer to sample resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10722,66 +10722,66 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_sample_resource *resource,
+			 struct mlx5_flow_dv_sample_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_sample = cache_resource->idx;
-	dev_flow->dv.sample_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_sample = resource->idx;
+	dev_flow->dv.sample_res = resource;
 	return 0;
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused,
-			    struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
-	if (resource->num_of_dest == cache_resource->num_of_dest &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    !memcmp((void *)cache_resource->sample_act,
-		    (void *)resource->sample_act,
-		   (resource->num_of_dest *
+	if (ctx_resource->num_of_dest == resource->num_of_dest &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    !memcmp((void *)resource->sample_act,
+		    (void *)ctx_resource->sample_act,
+		   (ctx_resource->num_of_dest *
 		   sizeof(struct mlx5_flow_sub_actions_list)))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
-		for (idx = 0; idx < resource->num_of_dest; idx++)
+		for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 			flow_dv_sample_sub_actions_release(dev,
-					&resource->sample_idx[idx]);
+					&ctx_resource->sample_idx[idx]);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
 	struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -10794,23 +10794,23 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	int ret;
 
 	/* Register new destination array resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
 					    &res_idx);
-	if (!cache_resource) {
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
 				 mlx5_malloc(MLX5_MEM_ZERO,
 				 sizeof(struct mlx5dv_dr_action_dest_attr),
@@ -10823,7 +10823,7 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 			goto error;
 		}
 		dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
-		sample_act = &resource->sample_act[idx];
+		sample_act = &ctx_resource->sample_act[idx];
 		action_flags = sample_act->action_flags;
 		switch (action_flags) {
 		case MLX5_FLOW_ACTION_QUEUE:
@@ -10854,9 +10854,9 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	/* create a dest array actioin */
 	ret = mlx5_os_flow_dr_create_flow_action_dest_array
 						(domain,
-						 cache_resource->num_of_dest,
+						 resource->num_of_dest,
 						 dest_attr,
-						 &cache_resource->action);
+						 &resource->action);
 	if (ret) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10864,19 +10864,18 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 				   "cannot create destination array action");
 		goto error;
 	}
-	cache_resource->idx = res_idx;
-	cache_resource->dev = dev;
-	for (idx = 0; idx < resource->num_of_dest; idx++)
+	resource->idx = res_idx;
+	resource->dev = dev;
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 		mlx5_free(dest_attr[idx]);
-	return &cache_resource->entry;
+	return &resource->entry;
 error:
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[idx]);
+						   &resource->sample_idx[idx]);
 		if (dest_attr[idx])
 			mlx5_free(dest_attr[idx]);
 	}
-
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
 	return NULL;
 }
@@ -10886,8 +10885,8 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to destination array resource.
+ * @param[in] ref
+ *   Pointer to destination array resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10898,25 +10897,25 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_dest_array_resource *resource,
+			 struct mlx5_flow_dv_dest_array_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_dest_array = cache_resource->idx;
-	dev_flow->dv.dest_array_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_dest_array = resource->idx;
+	dev_flow->dv.dest_array_res = resource;
 	return 0;
 }
 
@@ -13345,14 +13344,15 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			  struct mlx5_cache_entry *entry)
+flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+			  struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache),
-							  entry);
+	struct mlx5_flow_dv_matcher *resource = container_of(entry,
+							     typeof(*resource),
+							     entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object));
-	mlx5_free(cache);
+	claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
+	mlx5_free(resource);
 }
 
 /**
@@ -13376,7 +13376,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13395,7 +13395,7 @@ flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
-		container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
@@ -13417,15 +13417,14 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
 				     uint32_t encap_decap_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-					encap_decap_idx);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+				  encap_decap_idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->action);
-	return mlx5_hlist_unregister(priv->sh->encaps_decaps,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
 }
 
 /**
@@ -13487,15 +13486,15 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry)
+flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_port_id_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+				  container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
 
 /**
@@ -13514,14 +13513,14 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 					uint32_t port_id)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->port_id_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13544,15 +13543,15 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry)
+flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
 
 /**
@@ -13571,15 +13570,15 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 					  struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx = handle->dvh.rix_push_vlan;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->push_vlan_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13616,26 +13615,24 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry)
+flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	if (cache_resource->verbs_action)
+	if (resource->verbs_action)
 		claim_zero(mlx5_flow_os_destroy_flow_action
-				(cache_resource->verbs_action));
-	if (cache_resource->normal_path_tbl)
+						      (resource->verbs_action));
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-			cache_resource->normal_path_tbl);
-	flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			cache_resource->idx);
-	DRV_LOG(DEBUG, "sample resource %p: removed",
-		(void *)cache_resource);
+					     resource->normal_path_tbl);
+	flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
+	DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
 }
 
 /**
@@ -13654,38 +13651,36 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 				     struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			 handle->dvh.rix_sample);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+				  handle->dvh.rix_sample);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->verbs_action);
-	return mlx5_cache_unregister(&priv->sh->sample_action_list,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->verbs_action);
+	return mlx5_list_unregister(&priv->sh->sample_action_list,
+				    &resource->entry);
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	uint32_t i = 0;
 
-	MLX5_ASSERT(cache_resource->action);
-	if (cache_resource->action)
-		claim_zero(mlx5_flow_os_destroy_flow_action
-					(cache_resource->action));
-	for (; i < cache_resource->num_of_dest; i++)
+	MLX5_ASSERT(resource->action);
+	if (resource->action)
+		claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	for (; i < resource->num_of_dest; i++)
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[i]);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			cache_resource->idx);
+						   &resource->sample_idx[i]);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
 	DRV_LOG(DEBUG, "destination array resource %p: removed",
-		(void *)cache_resource);
+		(void *)resource);
 }
 
 /**
@@ -13704,15 +13699,15 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 				    struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_dest_array_resource *cache;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			       handle->dvh.rix_dest_array);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				  handle->dvh.rix_dest_array);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->dest_array_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->dest_array_list,
+				    &resource->entry);
 }
 
 static void
@@ -14555,7 +14550,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 		if (sub_policy->color_matcher[i]) {
 			tbl = container_of(sub_policy->color_matcher[i]->tbl,
 				typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 				      &sub_policy->color_matcher[i]->entry);
 			sub_policy->color_matcher[i] = NULL;
 		}
@@ -15289,8 +15284,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_cache_unregister(&tbl->matchers,
-				      &mtrmng->def_matcher[i]->entry);
+			mlx5_list_unregister(&tbl->matchers,
+					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
 		for (j = 0; j < MLX5_REG_BITS; j++) {
@@ -15299,8 +15294,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_cache_unregister(&tbl->matchers,
-					&mtrmng->drop_matcher[i][j]->entry);
+				mlx5_list_unregister(&tbl->matchers,
+					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
 		}
@@ -15396,7 +15391,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 			bool is_default_policy,
 			struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
@@ -15432,7 +15427,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -15795,7 +15790,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 						     0, &error);
 	uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
 	uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
 			.size = sizeof(matcher.mask.buf) -
@@ -15841,7 +15836,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -15878,7 +15873,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
@@ -16064,7 +16059,6 @@ flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
 	return NULL;
 }
 
-
 /**
  * Destroy the sub policy table with RX queue.
  *
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 1b264e5994..3dcc71d51d 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index bb9a908087..8395332507 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 0ed279e162..a2b5accb84 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,29 +9,29 @@
 #include "mlx5_utils.h"
 
 
-/********************* Cache list ************************/
+/********************* MLX5 list ************************/
 
-static struct mlx5_cache_entry *
-mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry __rte_unused,
+static struct mlx5_list_entry *
+mlx5_list_default_create_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry __rte_unused,
 			     void *ctx __rte_unused)
 {
 	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
 }
 
 static void
-mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
 }
 
 int
-mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
+mlx5_list_create(struct mlx5_list *list, const char *name,
 		     uint32_t entry_size, void *ctx,
-		     mlx5_cache_create_cb cb_create,
-		     mlx5_cache_match_cb cb_match,
-		     mlx5_cache_remove_cb cb_remove)
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove)
 {
 	MLX5_ASSERT(list);
 	if (!cb_match || (!cb_create ^ !cb_remove))
@@ -40,19 +40,19 @@ mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
+	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
+	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
 	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	LIST_INIT(&list->head);
 	return 0;
 }
 
-static struct mlx5_cache_entry *
-__cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	LIST_FOREACH(entry, &list->head, next) {
 		if (list->cb_match(list, entry, ctx))
@@ -60,7 +60,7 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 		if (reuse) {
 			__atomic_add_fetch(&entry->ref_cnt, 1,
 					   __ATOMIC_RELAXED);
-			DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
 		break;
@@ -68,33 +68,33 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_cache_entry *
-cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __cache_lookup(list, ctx, reuse);
+	entry = __list_lookup(list, ctx, reuse);
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	return cache_lookup(list, ctx, false);
+	return list_lookup(list, ctx, false);
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	uint32_t prev_gen_cnt = 0;
 
 	MLX5_ASSERT(list);
 	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
 	/* Lookup with read lock, reuse if found. */
-	entry = cache_lookup(list, ctx, true);
+	entry = list_lookup(list, ctx, true);
 	if (entry)
 		return entry;
 	/* Not found, append with write lock - block read from other threads. */
@@ -102,13 +102,13 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __cache_lookup(list, ctx, true);
+		entry = __list_lookup(list, ctx, true);
 		if (entry)
 			goto done;
 	}
 	entry = list->cb_create(list, entry, ctx);
 	if (!entry) {
-		DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
+		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
 			list->name, (void *)entry);
 		goto done;
 	}
@@ -116,7 +116,7 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	LIST_INSERT_HEAD(&list->head, entry, next);
 	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 done:
 	rte_rwlock_write_unlock(&list->lock);
@@ -124,12 +124,12 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 }
 
 int
-mlx5_cache_unregister(struct mlx5_cache_list *list,
-		      struct mlx5_cache_entry *entry)
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
 {
 	rte_rwlock_write_lock(&list->lock);
 	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 	if (--entry->ref_cnt) {
 		rte_rwlock_write_unlock(&list->lock);
@@ -140,15 +140,15 @@ mlx5_cache_unregister(struct mlx5_cache_list *list,
 	LIST_REMOVE(entry, next);
 	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 		list->name, (void *)entry);
 	return 0;
 }
 
 void
-mlx5_cache_list_destroy(struct mlx5_cache_list *list)
+mlx5_list_destroy(struct mlx5_list *list)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	MLX5_ASSERT(list);
 	/* no LIST_FOREACH_SAFE, using while instead */
@@ -156,14 +156,14 @@ mlx5_cache_list_destroy(struct mlx5_cache_list *list)
 		entry = LIST_FIRST(&list->head);
 		LIST_REMOVE(entry, next);
 		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
 			list->name, (void *)entry);
 	}
 	memset(list, 0, sizeof(*list));
 }
 
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
+mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
 	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 737dd7052d..593793345d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -296,19 +296,19 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ cache list *****************************/
+/************************ mlx5 list *****************************/
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
 
-struct mlx5_cache_list;
+struct mlx5_list;
 
 /**
- * Structure of the entry in the cache list, user should define its own struct
+ * Structure of the entry in the mlx5 list, user should define its own struct
  * that contains this in order to store the data.
  */
-struct mlx5_cache_entry {
-	LIST_ENTRY(mlx5_cache_entry) next; /* Entry pointers in the list. */
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* Reference count. */
 };
 
@@ -316,18 +316,18 @@ struct mlx5_cache_entry {
  * Type of callback function for entry removal.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
-				     struct mlx5_cache_entry *entry);
+typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
+				     struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -336,14 +336,14 @@ typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
-				   struct mlx5_cache_entry *entry, void *ctx);
+typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+				   struct mlx5_list_entry *entry, void *ctx);
 
 /**
- * Type of function for user defined cache list entry creation.
+ * Type of function for user defined mlx5 list entry creation.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The new allocated entry, NULL if list entry size unspecified,
  *   New entry has to be allocated in callback and return.
@@ -353,46 +353,46 @@ typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_cache_entry *(*mlx5_cache_create_cb)
-				 (struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry,
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry,
 				  void *ctx);
 
 /**
- * Linked cache list structure.
+ * Linked mlx5 list structure.
  *
- * Entry in cache list could be reused if entry already exists,
+ * Entry in mlx5 list could be reused if entry already exists,
  * reference count will increase and the existing entry returns.
  *
  * When destroy an entry from list, decrease reference count and only
  * destroy when no further reference.
  *
- * Linked list cache is designed for limited number of entries cache,
+ * Linked list is designed for limited number of entries,
  * read mostly, less modification.
  *
- * For huge amount of entries cache, please consider hash list cache.
+ * For huge amount of entries, please consider hash list.
  *
  */
-struct mlx5_cache_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the cache list. */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	mlx5_cache_create_cb cb_create; /**< entry create callback. */
-	mlx5_cache_match_cb cb_match; /**< entry match callback. */
-	mlx5_cache_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_cache_head, mlx5_cache_entry) head;
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
 };
 
 /**
- * Initialize a cache list.
+ * Create a mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
  * @param name
- *   Name of the cache list.
+ *   Name of the mlx5 list.
  * @param entry_size
  *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
@@ -406,11 +406,11 @@ struct mlx5_cache_list {
  * @return
  *   0 on success, otherwise failure.
  */
-int mlx5_cache_list_init(struct mlx5_cache_list *list,
+int mlx5_list_create(struct mlx5_list *list,
 			 const char *name, uint32_t entry_size, void *ctx,
-			 mlx5_cache_create_cb cb_create,
-			 mlx5_cache_match_cb cb_match,
-			 mlx5_cache_remove_cb cb_remove);
+			 mlx5_list_create_cb cb_create,
+			 mlx5_list_match_cb cb_match,
+			 mlx5_list_remove_cb cb_remove);
 
 /**
  * Search an entry matching the key.
@@ -419,18 +419,18 @@ int mlx5_cache_list_init(struct mlx5_cache_list *list,
  * this function only in main thread.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  * @param ctx
  *   Common context parameter used by entry callback function.
  *
  * @return
- *   Pointer of the cache entry if found, NULL otherwise.
+ *   Pointer of the list entry if found, NULL otherwise.
  */
-struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
 					   void *ctx);
 
 /**
- * Reuse or create an entry to the cache list.
+ * Reuse or create an entry to the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
@@ -440,42 +440,42 @@ struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
  * @return
  *   registered entry on success, NULL otherwise
  */
-struct mlx5_cache_entry *mlx5_cache_register(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
 					     void *ctx);
 
 /**
- * Remove an entry from the cache list.
+ * Remove an entry from the mlx5 list.
  *
  * User should guarantee the validity of the entry.
  *
  * @param list
  *   Pointer to the hast list.
  * @param entry
- *   Entry to be removed from the cache list table.
+ *   Entry to be removed from the mlx5 list table.
  * @return
  *   0 on entry removed, 1 on entry still referenced.
  */
-int mlx5_cache_unregister(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry);
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
 
 /**
- * Destroy the cache list.
+ * Destroy the mlx5 list.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  */
-void mlx5_cache_list_destroy(struct mlx5_cache_list *list);
+void mlx5_list_destroy(struct mlx5_list *list);
 
 /**
- * Get entry number from the cache list.
+ * Get entry number from the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list.
  * @return
- *   Cache list entry number.
+ *   mlx5 list entry number.
  */
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list);
+mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /********************************* indexed pool *************************/
 
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 7d15c998bb..b10c47fee3 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -608,10 +608,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v2 07/22] net/mlx5: add per lcore cache to the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When mlx5 list object is accessed by multiple cores, the list lock
counter is all the time written by all the cores what increases cache
misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup
operation, all the other threads coming to do an operation in the list
are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when
the list objects are mostly reused.

Synchronization with atomic operations should be done in order to
allow threads to unregister an entry from other thread cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 9aa57e38b7..8a043526da 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1702,10 +1710,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 803387b98e..b01ce5cbfd 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1365,6 +1365,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 4dec703366..ce363355c1 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1634,7 +1634,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1642,6 +1646,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1650,6 +1659,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1658,6 +1672,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index ed0381329f..4a86d153f0 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3820,6 +3820,39 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_port_id_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate port_id action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
+}
+
 /**
  * Find existing table port ID resource or create and register a new one.
  *
@@ -3912,6 +3945,39 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate push_vlan action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
+}
+
 /**
  * Find existing push vlan resource or create and register a new one.
  *
@@ -9848,6 +9914,36 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 	return match_criteria_enable;
 }
 
+static struct mlx5_list_entry *
+flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_matcher *ref = ctx->data;
+	struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
+							    typeof(*tbl), tbl);
+	struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
+							    sizeof(*resource),
+							    0, SOCKET_ID_ANY);
+
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot create matcher");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->tbl = &tbl->tbl;
+	return &resource->entry;
+}
+
+static void
+flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 struct mlx5_hlist_entry *
 flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 {
@@ -9914,10 +10010,12 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
 			 flow_dv_matcher_create_cb,
 			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb);
+			 flow_dv_matcher_remove_cb,
+			 flow_dv_matcher_clone_cb,
+			 flow_dv_matcher_clone_free_cb);
 	return &tbl_data->entry;
 }
 
@@ -10705,6 +10803,45 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 
 }
 
+struct mlx5_list_entry *
+flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t idx = 0;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_sample_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+			resource->idx);
+}
+
 /**
  * Find existing sample resource or create and register a new one.
  *
@@ -10880,6 +11017,46 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 	return NULL;
 }
 
+struct mlx5_list_entry *
+flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t res_idx = 0;
+	struct rte_flow_error *error = ctx->error;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				      &res_idx);
+	if (!resource) {
+		rte_flow_error_set(error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate dest-array memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = res_idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
+}
+
 /**
  * Find existing destination array resource or create and register a new one.
  *
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 3dcc71d51d..5450ddd388 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 8395332507..f8769da8dc 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1857,20 +1857,18 @@ mlx5_ind_table_obj_get(struct rte_eth_dev *dev, const uint16_t *queues,
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_ind_table_obj *ind_tbl;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		if ((ind_tbl->queues_n == queues_n) &&
 		    (memcmp(ind_tbl->queues, queues,
 			    ind_tbl->queues_n * sizeof(ind_tbl->queues[0]))
-		     == 0))
+		     == 0)) {
+			__atomic_fetch_add(&ind_tbl->refcnt, 1,
+					   __ATOMIC_RELAXED);
 			break;
+		}
 	}
-	if (ind_tbl) {
-		unsigned int i;
-
-		__atomic_fetch_add(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
-		for (i = 0; i != ind_tbl->queues_n; ++i)
-			mlx5_rxq_get(dev, ind_tbl->queues[i]);
-	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ind_tbl;
 }
 
@@ -1893,19 +1891,20 @@ mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
 			   bool standalone)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	unsigned int i;
+	unsigned int i, ret;
 
-	if (__atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED) == 0)
-		priv->obj_ops.ind_table_destroy(ind_tbl);
+	rte_rwlock_write_lock(&priv->ind_tbls_lock);
+	ret = __atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
+	if (!ret && !standalone)
+		LIST_REMOVE(ind_tbl, next);
+	rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	if (ret)
+		return 1;
+	priv->obj_ops.ind_table_destroy(ind_tbl);
 	for (i = 0; i != ind_tbl->queues_n; ++i)
 		claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
-	if (__atomic_load_n(&ind_tbl->refcnt, __ATOMIC_RELAXED) == 0) {
-		if (!standalone)
-			LIST_REMOVE(ind_tbl, next);
-		mlx5_free(ind_tbl);
-		return 0;
-	}
-	return 1;
+	mlx5_free(ind_tbl);
+	return 0;
 }
 
 /**
@@ -1924,12 +1923,14 @@ mlx5_ind_table_obj_verify(struct rte_eth_dev *dev)
 	struct mlx5_ind_table_obj *ind_tbl;
 	int ret = 0;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		DRV_LOG(DEBUG,
 			"port %u indirection table obj %p still referenced",
 			dev->data->port_id, (void *)ind_tbl);
 		++ret;
 	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ret;
 }
 
@@ -2015,8 +2016,11 @@ mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues,
 		mlx5_free(ind_tbl);
 		return NULL;
 	}
-	if (!standalone)
+	if (!standalone) {
+		rte_rwlock_write_lock(&priv->ind_tbls_lock);
 		LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
+		rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	}
 	return ind_tbl;
 }
 
@@ -2328,6 +2332,35 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 	return hrxq ? &hrxq->entry : NULL;
 }
 
+struct mlx5_list_entry *
+mlx5_hrxq_clone_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry,
+		    void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq;
+	uint32_t hrxq_idx = 0;
+
+	hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx);
+	if (!hrxq)
+		return NULL;
+	memcpy(hrxq, entry, sizeof(*hrxq) + MLX5_RSS_HASH_KEY_LEN);
+	hrxq->idx = hrxq_idx;
+	return &hrxq->entry;
+}
+
+void
+mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq->idx);
+}
+
 /**
  * Get an Rx Hash queue.
  *
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index a2b5accb84..51cca68ea9 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,57 +9,68 @@
 #include "mlx5_utils.h"
 
 
-/********************* MLX5 list ************************/
-
-static struct mlx5_list_entry *
-mlx5_list_default_create_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
-{
-	mlx5_free(entry);
-}
+/********************* mlx5 list ************************/
 
 int
-mlx5_list_create(struct mlx5_list *list, const char *name,
-		     uint32_t entry_size, void *ctx,
-		     mlx5_list_create_cb cb_create,
-		     mlx5_list_match_cb cb_match,
-		     mlx5_list_remove_cb cb_remove)
+mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	int i;
+
 	MLX5_ASSERT(list);
-	if (!cb_match || (!cb_create ^ !cb_remove))
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free)
 		return -1;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
+	list->cb_create = cb_create;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	LIST_INIT(&list->head);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry;
-
-	LIST_FOREACH(entry, &list->head, next) {
-		if (list->cb_match(list, entry, ctx))
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (list->cb_match(list, entry, ctx)) {
+			if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_ACQUIRE);
+				if (ret == 0) {
+					LIST_REMOVE(entry, next);
+					list->cb_clone_free(list, entry);
+				}
+			}
+			entry = nentry;
 			continue;
+		}
 		if (reuse) {
-			__atomic_add_fetch(&entry->ref_cnt, 1,
-					   __ATOMIC_RELAXED);
+			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+						 __ATOMIC_ACQUIRE);
+			if (ret == 1u) {
+				/* Entry was invalid before, free it. */
+				LIST_REMOVE(entry, next);
+				list->cb_clone_free(list, entry);
+				entry = nentry;
+				continue;
+			}
 			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
@@ -68,96 +79,141 @@ __list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_list_entry *
-list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry = NULL;
+	int i;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, ctx, reuse);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	return list_lookup(list, ctx, false);
+	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+
+	if (!lentry)
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
 }
 
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry, *lentry;
 	uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
-	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* Lookup in local cache. */
+	lentry = __list_lookup(list, lcore_index, ctx, true);
+	if (lentry)
+		return lentry;
 	/* Lookup with read lock, reuse if found. */
-	entry = list_lookup(list, ctx, true);
-	if (entry)
-		return entry;
+	rte_rwlock_read_lock(&list->lock);
+	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+	if (entry == NULL) {
+		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
+					       __ATOMIC_ACQUIRE);
+		rte_rwlock_read_unlock(&list->lock);
+	} else {
+		rte_rwlock_read_unlock(&list->lock);
+		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+	}
 	/* Not found, append with write lock - block read from other threads. */
 	rte_rwlock_write_lock(&list->lock);
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, ctx, true);
-		if (entry)
-			goto done;
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (entry) {
+			rte_rwlock_write_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
 	}
 	entry = list->cb_create(list, entry, ctx);
-	if (!entry) {
-		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
-			list->name, (void *)entry);
-		goto done;
+	if (entry) {
+		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+		if (!lentry) {
+			list->cb_remove(list, entry);
+		} else {
+			entry->ref_cnt = 1u;
+			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
+					 next);
+			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
+			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+				list->name, (void *)entry, entry->ref_cnt);
+		}
+
 	}
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(&list->head, entry, next);
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-done:
 	rte_rwlock_write_unlock(&list->lock);
-	return entry;
+	return lentry;
 }
 
 int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
+	struct mlx5_list_entry *gentry = entry->gentry;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
+	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
+		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		LIST_REMOVE(gentry, next);
+		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
-		return 1;
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
 	}
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-	__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	LIST_REMOVE(entry, next);
-	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-		list->name, (void *)entry);
-	return 0;
+	return 1;
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
+	int i;
 
 	MLX5_ASSERT(list);
-	/* no LIST_FOREACH_SAFE, using while instead */
-	while (!LIST_EMPTY(&list->head)) {
-		entry = LIST_FIRST(&list->head);
-		LIST_REMOVE(entry, next);
-		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
-			list->name, (void *)entry);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list, entry);
+			}
+		}
 	}
 	memset(list, 0, sizeof(*list));
 }
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 593793345d..24ae2b2ccb 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -309,9 +309,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -339,6 +344,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -375,15 +387,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -393,8 +407,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -407,10 +419,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index b10c47fee3..f6cf1928b2 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -608,9 +608,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v2 08/22] net/mlx5: minimize list critical sections

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is
taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical
sections.

These operations are heavy and make the critical sections heavy because
they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation
counter in order to detect parallel allocations.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 51cca68ea9..772b352af5 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 24ae2b2ccb..684d1e8a2a 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -387,8 +387,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
-- 
2.25.1

[dpdk-dev] [PATCH v2 09/22] net/mlx5: manage list cache entries release

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When a cache entry is allocated by lcore A and is released by lcore B,
the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be
increased atomically when the non-original lcore decreases the reference
counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in
its cache, it will check the counter in order to free invalid entries in
its cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 772b352af5..7cdf44dcf7 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 684d1e8a2a..ffa9cd5142 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -310,11 +310,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v2 10/22] net/mlx5: relax the list utility atomic operations

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The atomic operation in the list utility no need a barriers because the
critical part are managed by RW lock.

Relax them.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 7cdf44dcf7..29248c80ed 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
-- 
2.25.1

[dpdk-dev] [PATCH v2 11/22] net/mlx5: allocate list memory by the create API

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence
with the hlist utility.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  56 ++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 137 insertions(+), 81 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 8a043526da..87b63d852b 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -447,6 +455,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -508,9 +532,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1710,11 +1748,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1771,7 +1811,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 9aade013c5..775ea15adb 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1611,7 +1611,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index b01ce5cbfd..9319604d91 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ce363355c1..ce4d205e86 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 4a86d153f0..4aafe87524 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3883,7 +3883,7 @@ flow_dv_port_id_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -4008,7 +4008,7 @@ flow_dv_push_vlan_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10010,12 +10010,22 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
-			 flow_dv_matcher_create_cb,
-			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb,
-			 flow_dv_matcher_clone_cb,
-			 flow_dv_matcher_clone_free_cb);
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+					      flow_dv_matcher_create_cb,
+					      flow_dv_matcher_match_cb,
+					      flow_dv_matcher_remove_cb,
+					      flow_dv_matcher_clone_cb,
+					      flow_dv_matcher_clone_free_cb);
+	if (!tbl_data->matchers) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot create tbl matcher list");
+		mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
+		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
+		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
+		return NULL;
+	}
 	return &tbl_data->entry;
 }
 
@@ -10143,7 +10153,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10275,7 +10285,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
@@ -10872,7 +10882,7 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -11087,7 +11097,7 @@ flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -13553,7 +13563,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13696,7 +13706,7 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+	return mlx5_list_unregister(priv->sh->port_id_action_list,
 				    &resource->entry);
 }
 
@@ -13754,7 +13764,7 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+	return mlx5_list_unregister(priv->sh->push_vlan_action_list,
 				    &resource->entry);
 }
 
@@ -13835,7 +13845,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->verbs_action);
-	return mlx5_list_unregister(&priv->sh->sample_action_list,
+	return mlx5_list_unregister(priv->sh->sample_action_list,
 				    &resource->entry);
 }
 
@@ -13883,7 +13893,7 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->dest_array_list,
+	return mlx5_list_unregister(priv->sh->dest_array_list,
 				    &resource->entry);
 }
 
@@ -14727,7 +14737,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 		if (sub_policy->color_matcher[i]) {
 			tbl = container_of(sub_policy->color_matcher[i]->tbl,
 				typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 				      &sub_policy->color_matcher[i]->entry);
 			sub_policy->color_matcher[i] = NULL;
 		}
@@ -15461,7 +15471,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
@@ -15471,7 +15481,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_list_unregister(&tbl->matchers,
+				mlx5_list_unregister(tbl->matchers,
 					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
@@ -15604,7 +15614,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -16013,7 +16023,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -16050,7 +16060,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index f8769da8dc..aa9e973d10 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 29248c80ed..a4526444f9 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index ffa9cd5142..0bf2f5f5ca 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -419,15 +419,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index f6cf1928b2..97a8f04e39 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -608,7 +608,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v2 12/22] common/mlx5: add per-lcore cache to hash list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates
all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and
hlist.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 doc/guides/nics/mlx5.rst                |   5 +
 doc/guides/rel_notes/release_21_08.rst  |   6 +
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 466 +++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h | 261 +++++++++----
 drivers/net/mlx5/linux/mlx5_os.c        |  46 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++---
 drivers/net/mlx5/mlx5_flow.h            | 185 +++++-----
 drivers/net/mlx5/mlx5_flow_dv.c         | 407 ++++++++++++---------
 drivers/net/mlx5/mlx5_rx.h              |  13 +-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +--
 drivers/net/mlx5/mlx5_utils.c           | 251 -------------
 drivers/net/mlx5/mlx5_utils.h           | 197 ----------
 drivers/net/mlx5/windows/mlx5_os.c      |   2 +-
 16 files changed, 1016 insertions(+), 1044 deletions(-)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index eb44a070b1..9bd3846e0d 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -445,6 +445,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..f6cd1d137d 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Updated Mellanox mlx5 driver.**
+
+  Updated the Mellanox mlx5 driver with new features and improvements, including:
+
+  * Optimize multiple-thread flow insertion rate.
+
 
 Removed Items
 -------------
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..1809ff1e95 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index ad2011e858..4e385c616a 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -11,39 +11,324 @@
 #include "mlx5_common_utils.h"
 #include "mlx5_common_log.h"
 
-/********************* Hash List **********************/
+/********************* mlx5 list ************************/
+
+static int
+mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
+	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       mlx5_list_match_cb cb_match,
+	       mlx5_list_remove_cb cb_remove,
+	       mlx5_list_clone_cb cb_clone,
+	       mlx5_list_clone_free_cb cb_clone_free)
+{
+	int i;
+
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return -EINVAL;
+	}
+	if (name)
+		snprintf(list->name, sizeof(list->name), "%s", name);
+	list->ctx = ctx;
+	list->lcores_share = lcores_share;
+	list->cb_create = cb_create;
+	list->cb_match = cb_match;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
+	rte_rwlock_init(&list->lock);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
+	return 0;
+}
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (mlx5_list_init(list, name, ctx, lcores_share,
+			   cb_create, cb_match, cb_remove, cb_clone,
+			   cb_clone_free) != 0) {
+		mlx5_free(list);
+		return NULL;
+	}
+	return list;
+}
+
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+{
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_RELAXED) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_RELAXED);
+			}
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
+		}
+		entry = LIST_NEXT(entry, next);
+	}
+	return NULL;
+}
+
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry = NULL;
+	int i;
+
+	rte_rwlock_read_lock(&list->lock);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
+	rte_rwlock_read_unlock(&list->lock);
+	return entry;
+}
+
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
+{
+	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+
+	if (unlikely(!lentry))
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
+}
+
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			if (list->lcores_share)
+				list->cb_clone_free(list->ctx, entry);
+			else
+				list->cb_remove(list->ctx, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	MLX5_ASSERT(list);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
+		rte_rwlock_read_unlock(&list->lock);
+	}
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list->ctx, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+		return entry;
+	}
+	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list->ctx, entry);
+		return NULL;
+	}
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
+	rte_rwlock_write_lock(&list->lock);
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
+			rte_rwlock_write_unlock(&list->lock);
+			list->cb_remove(list->ctx, entry);
+			list->cb_clone_free(list->ctx, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
+		}
+	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
+	rte_rwlock_write_unlock(&list->lock);
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
+	return local_entry;
+}
 
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
 {
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
+	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		if (list->lcores_share)
+			list->cb_clone_free(list->ctx, entry);
+		else
+			list->cb_remove(list->ctx, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	rte_rwlock_write_lock(&list->lock);
+	if (likely(gentry->ref_cnt == 0)) {
+		LIST_REMOVE(gentry, next);
+		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list->ctx, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
+	}
+	rte_rwlock_write_unlock(&list->lock);
+	return 1;
 }
 
 static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
+mlx5_list_uninit(struct mlx5_list *list)
+{
+	struct mlx5_list_entry *entry;
+	int i;
+
+	MLX5_ASSERT(list);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list->ctx, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list->ctx, entry);
+			}
+		}
+	}
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	mlx5_list_uninit(list);
+	mlx5_free(list);
+}
+
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list)
 {
-	mlx5_free(entry);
+	MLX5_ASSERT(list);
+	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
 }
 
+/********************* Hash List **********************/
+
 struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
+		  bool lcores_share, void *ctx, mlx5_list_create_cb cb_create,
+		  mlx5_list_match_cb cb_match,
+		  mlx5_list_remove_cb cb_remove,
+		  mlx5_list_clone_cb cb_clone,
+		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+		DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
+			"be aligned to 0x%" PRIX32 ".", size, act_size);
 	} else {
 		act_size = size;
 	}
@@ -57,61 +342,24 @@ mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 			name ? name : "None");
 		return NULL;
 	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
 	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
+	h->lcores_share = lcores_share;
+	h->direct_key = direct_key;
+	for (i = 0; i < act_size; i++) {
+		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   cb_create, cb_match, cb_remove, cb_clone,
+				   cb_clone_free) != 0) {
+			mlx5_free(h);
+			return NULL;
 		}
 	}
-	return node;
+	DRV_LOG(DEBUG, "Hash list %s with size 0x%" PRIX32 " was created.",
+		name, act_size);
+	return h;
 }
 
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
 
-struct mlx5_hlist_entry *
+struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
@@ -120,102 +368,44 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
+	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
 }
 
-struct mlx5_hlist_entry*
+struct mlx5_list_entry*
 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry;
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
+	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	if (likely(entry)) {
+		if (h->lcores_share)
+			entry->gentry->bucket_idx = idx;
+		else
+			entry->bucket_idx = idx;
 	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
-	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
 	return entry;
 }
 
 int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = entry->idx;
+	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+							      entry->bucket_idx;
 
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
+	return mlx5_list_unregister(&h->buckets[idx].l, entry);
 }
 
 void
 mlx5_hlist_destroy(struct mlx5_hlist *h)
 {
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
+	uint32_t i;
 
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
+	for (i = 0; i <= h->mask; i++)
+		mlx5_list_uninit(&h->buckets[i].l);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index ed378ce9bd..79d0bc1a50 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -7,106 +7,221 @@
 
 #include "mlx5_common.h"
 
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
+/************************ mlx5 list *****************************/
 
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE			32
+/** Maximum size of string for naming. */
+#define MLX5_NAME_SIZE			32
 
-struct mlx5_hlist;
+struct mlx5_list;
 
 /**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
+ * Structure of the entry in the mlx5 list, user should define its own struct
+ * that contains this in order to store the data.
  */
-struct mlx5_hlist_entry {
-	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint32_t idx; /* Bucket index the entry belongs to. */
-	uint32_t ref_cnt; /* Reference count. */
-};
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
+	uint32_t ref_cnt __rte_aligned(8); /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
+	union {
+		struct mlx5_list_entry *gentry;
+		uint32_t bucket_idx;
+	};
+} __rte_packed;
 
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
+} __rte_cache_aligned;
 
 /**
  * Type of callback function for entry removal.
  *
- * @param list
- *   The hash list.
+ * @param tool_ctx
+ *   The tool instance user context.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
-				     struct mlx5_hlist_entry *entry);
+typedef void (*mlx5_list_remove_cb)(void *tool_ctx,
+				    struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
- * @param list
- *   The hash list.
+ * @param tool_ctx
+ *   The tool instance context.
  * @param entry
  *   The entry in the list.
- * @param key
- *   The new entry key.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry,
-				   uint64_t key, void *ctx);
+typedef int (*mlx5_list_match_cb)(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *ctx);
+
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)(void *tool_ctx,
+				      struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
+					struct mlx5_list_entry *entry);
 
 /**
- * Type of function for user defined hash list entry creation.
+ * Type of function for user defined mlx5 list entry creation.
  *
- * @param list
- *   The hash list.
- * @param key
- *   The key of the new entry.
+ * @param tool_ctx
+ *   The mlx5 tool instance context.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
- *   Pointer to allocated entry on success, NULL otherwise.
+ *   Pointer of entry on success, NULL otherwise.
+ */
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
+						       void *ctx);
+
+/**
+ * Linked mlx5 list structure.
+ *
+ * Entry in mlx5 list could be reused if entry already exists,
+ * reference count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ *
+ * Linked list is designed for limited number of entries,
+ * read mostly, less modification.
+ *
+ * For huge amount of entries, please consider hash list.
+ *
+ */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	void *ctx; /* user objects target to callback. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
+};
+
+/**
+ * Create a mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param name
+ *   Name of the mlx5 list.
+ * @param ctx
+ *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
+ * @param cb_create
+ *   Callback function for entry create.
+ * @param cb_match
+ *   Callback function for entry match.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @return
+ *   List pointer on success, otherwise NULL.
+ */
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ * @param ctx
+ *   Common context parameter used by entry callback function.
+ *
+ * @return
+ *   Pointer of the list entry if found, NULL otherwise.
+ */
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
+					   void *ctx);
+
+/**
+ * Reuse or create an entry to the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param ctx
+ *   Common context parameter used by callback function.
+ *
+ * @return
+ *   registered entry on success, NULL otherwise
+ */
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
+					     void *ctx);
+
+/**
+ * Remove an entry from the mlx5 list.
+ *
+ * User should guarantee the validity of the entry.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @param entry
+ *   Entry to be removed from the mlx5 list table.
+ * @return
+ *   0 on entry removed, 1 on entry still referenced.
  */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
-				  (struct mlx5_hlist *list,
-				   uint64_t key, void *ctx);
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
 
-/* Hash list bucket head. */
+/**
+ * Destroy the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ */
+void mlx5_list_destroy(struct mlx5_list *list);
+
+/**
+ * Get entry number from the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @return
+ *   mlx5 list entry number.
+ */
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list);
+
+/********************* Hash List **********************/
+
+/* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_hlist_head head; /* List head. */
-	rte_rwlock_t lock; /* Bucket lock. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
+	struct mlx5_list l;
 } __rte_cache_aligned;
 
 /**
  * Hash list table structure
  *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
+ * The hash list bucket using the mlx5_list object for managing.
  */
 struct mlx5_hlist {
-	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
-	/**< number of heads, need to be power of 2. */
-	uint32_t table_sz;
-	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
-	/**< mask to get the index of the list heads. */
-	uint32_t mask;
-	bool direct_key; /* Use the new entry key directly as hash index. */
-	bool write_most; /* List mostly used for append new or destroy. */
-	void *ctx;
-	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
-	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
-	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+	uint32_t mask; /* A mask for the bucket index range. */
+	uint8_t flags;
+	bool direct_key; /* Whether to use the key directly as hash index. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
-	/**< list bucket arrays. */
 };
 
 /**
@@ -123,23 +238,33 @@ struct mlx5_hlist {
  *   Heads array size of the hash list.
  * @param entry_size
  *   Entry size to allocate if cb_create not specified.
- * @param flags
- *   The hash list attribute flags.
+ * @param direct key
+ *   Whether to use the key directly as hash index.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
+ * @param ctx
+ *   The hlist instance context.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
  *   Callback function for entry match.
- * @param cb_destroy
- *   Callback function for entry destroy.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @param cb_clone
+ *   Callback function for entry clone.
+ * @param cb_clone_free
+ *   Callback function for entry clone free.
  * @return
  *   Pointer of the hash list table created, NULL on failure.
  */
 __rte_internal
 struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
-				     uint32_t entry_size, uint32_t flags,
-				     mlx5_hlist_create_cb cb_create,
-				     mlx5_hlist_match_cb cb_match,
-				     mlx5_hlist_remove_cb cb_destroy);
+				     bool direct_key, bool lcores_share,
+				     void *ctx, mlx5_list_create_cb cb_create,
+				     mlx5_list_match_cb cb_match,
+				     mlx5_list_remove_cb cb_remove,
+				     mlx5_list_clone_cb cb_clone,
+				     mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
@@ -158,7 +283,7 @@ struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
  *   Pointer of the hlist entry if found, NULL otherwise.
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
 					   void *ctx);
 
 /**
@@ -177,7 +302,7 @@ struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
  *   registered entry on success, NULL otherwise
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
 					     void *ctx);
 
 /**
@@ -192,7 +317,7 @@ struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
  *   0 on entry removed, 1 on entry still referenced.
  */
 __rte_internal
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry);
 
 /**
  * Destroy the hash list table, all the entries already inserted into the lists
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 87b63d852b..cf573a9a4d 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -261,7 +261,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+	char s[MLX5_NAME_SIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 						   flow_dv_port_id_create_cb,
 						   flow_dv_port_id_match_cb,
 						   flow_dv_port_id_remove_cb,
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 						    flow_dv_push_vlan_create_cb,
 						    flow_dv_push_vlan_match_cb,
 						    flow_dv_push_vlan_remove_cb,
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 						  flow_dv_sample_remove_cb,
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -314,44 +314,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb,
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
+					  false, sh, flow_dv_tag_create_cb,
 					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb);
+					  flow_dv_tag_remove_cb,
+					  flow_dv_tag_clone_cb,
+					  flow_dv_tag_clone_free_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, MLX5_HLIST_WRITE_MOST |
-					    MLX5_HLIST_DIRECT_KEY,
+					    true, false, sh,
 					    flow_dv_modify_create_cb,
 					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb);
+					    flow_dv_modify_remove_cb,
+					    flow_dv_modify_clone_cb,
+					    flow_dv_modify_clone_free_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, MLX5_HLIST_DIRECT_KEY |
-					      MLX5_HLIST_WRITE_MOST,
+					      true, true, sh,
 					      flow_dv_encap_decap_create_cb,
 					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb);
+					      flow_dv_encap_decap_remove_cb,
+					      flow_dv_encap_decap_clone_cb,
+					     flow_dv_encap_decap_clone_free_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -1748,7 +1748,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
 				       mlx5_hrxq_clone_cb,
@@ -1780,15 +1781,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
-						      0, 0,
+						      false, true, eth_dev,
 						      flow_dv_mreg_create_cb,
 						      flow_dv_mreg_match_cb,
-						      flow_dv_mreg_remove_cb);
+						      flow_dv_mreg_remove_cb,
+						      flow_dv_mreg_clone_cb,
+						    flow_dv_mreg_clone_free_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
-		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 775ea15adb..0e80408511 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1358,20 +1358,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 9319604d91..896c0c2da1 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 20ce0ed424..e6b71a87a0 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3983,28 +3983,27 @@ flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
-flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
-		      struct mlx5_hlist_entry *entry,
-		      uint64_t key, void *cb_ctx __rte_unused)
+flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
+		      struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
 
-	return mcp_res->mark_id != key;
+	return mcp_res->mark_id != *(uint32_t *)(ctx->data);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
-		       void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
 	struct rte_flow_error *error = ctx->error;
 	uint32_t idx = 0;
 	int ret;
-	uint32_t mark_id = key;
+	uint32_t mark_id = *(uint32_t *)(ctx->data);
 	struct rte_flow_attr attr = {
 		.group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
 		.ingress = 1,
@@ -4110,6 +4109,36 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return &mcp_res->hlist_ent;
 }
 
+struct mlx5_list_entry *
+flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	uint32_t idx = 0;
+
+	mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
+	if (!mcp_res) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	memcpy(mcp_res, oentry, sizeof(*mcp_res));
+	mcp_res->idx = idx;
+	return &mcp_res->hlist_ent;
+}
+
+void
+flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_mreg_copy_resource *mcp_res =
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
+}
+
 /**
  * Add a flow of copying flow metadata registers in RX_CP_TBL.
  *
@@ -4140,10 +4169,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 			  struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
+		.data = &mark_id,
 	};
 
 	/* Check if already registered. */
@@ -4156,11 +4186,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 }
 
 void
-flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
+flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
-	struct rte_eth_dev *dev = list->ctx;
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
@@ -4206,14 +4236,17 @@ flow_mreg_del_copy_action(struct rte_eth_dev *dev,
 static void
 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
 {
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check if default flow is registered. */
 	if (!priv->mreg_cp_tbl)
 		return;
-	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
-				  MLX5_DEFAULT_COPY_ID, NULL);
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
 	if (!entry)
 		return;
 	mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
@@ -4239,6 +4272,8 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check whether extensive metadata feature is engaged. */
 	if (!priv->config.dv_flow_en ||
@@ -4250,9 +4285,11 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 	 * Add default mreg copy flow may be called multiple time, but
 	 * only be called once in stop. Avoid register it twice.
 	 */
-	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
 		return 0;
-	mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
+	mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
 	if (!mcp_res)
 		return -rte_errno;
 	return 0;
@@ -8350,7 +8387,7 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	union tunnel_offload_mark mbits = { .val = mark };
 	union mlx5_flow_tbl_key table_key = {
 		{
@@ -8362,16 +8399,20 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 			.is_egress = 0,
 		}
 	};
-	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &table_key.v64,
+	};
+
+	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
 	return he ?
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
 
 static void
-mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry)
+mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
@@ -8380,26 +8421,26 @@ mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
 }
 
 static int
-mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-				  struct mlx5_hlist_entry *entry,
-				  uint64_t key, void *cb_ctx __rte_unused)
+mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
 }
 
-static struct mlx5_hlist_entry *
-mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
-				   void *ctx __rte_unused)
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 
 	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
@@ -8428,13 +8469,36 @@ mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return NULL;
 }
 
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *oentry,
+				  void *cb_ctx __rte_unused)
+{
+	struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
+						   0, SOCKET_ID_ANY);
+
+	if (!tte)
+		return NULL;
+	memcpy(tte, oentry, sizeof(*tte));
+	return &tte->hash;
+}
+
+static void
+mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
+				       struct mlx5_list_entry *entry)
+{
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	mlx5_free(tte);
+}
+
 static uint32_t
 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 				const struct mlx5_flow_tunnel *tunnel,
 				uint32_t group, uint32_t *table,
 				struct rte_flow_error *error)
 {
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key key = {
 		.tunnel_id = tunnel ? tunnel->tunnel_id : 0,
@@ -8442,9 +8506,12 @@ tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 	};
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_hlist *group_hash;
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &key.val,
+	};
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_register(group_hash, key.val, NULL);
+	he = mlx5_hlist_register(group_hash, key.val, &ctx);
 	if (!he)
 		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
@@ -8558,15 +8625,17 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
-					   mlx5_flow_tunnel_grp2tbl_remove_cb);
+					   mlx5_flow_tunnel_grp2tbl_remove_cb,
+					   mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(ipool, id);
 		return NULL;
 	}
-	tunnel->groups->ctx = priv->sh;
 	/* initiate new PMD tunnel */
 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
 	tunnel->tunnel_id = id;
@@ -8666,15 +8735,17 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES), 0,
-					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
+					 rte_align32pow2(MLX5_MAX_TABLES),
+					 false, true, sh,
+					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-					 mlx5_flow_tunnel_grp2tbl_remove_cb);
+					 mlx5_flow_tunnel_grp2tbl_remove_cb,
+					 mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
 	}
-	thub->groups->ctx = sh;
 	sh->tunnel_hub = thub;
 
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ce4d205e86..ab4e8c5c4f 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -480,7 +480,7 @@ struct mlx5_flow_dv_matcher {
 
 /* Encap/decap resource structure. */
 struct mlx5_flow_dv_encap_decap_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/* Pointer to next element. */
 	uint32_t refcnt; /**< Reference counter. */
 	void *action;
@@ -495,7 +495,7 @@ struct mlx5_flow_dv_encap_decap_resource {
 
 /* Tag resource structure. */
 struct mlx5_flow_dv_tag_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry for tag resource, tag value as the key. */
 	void *action;
 	/**< Tag action object. */
@@ -519,7 +519,7 @@ struct mlx5_flow_dv_tag_resource {
 
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
@@ -569,7 +569,7 @@ struct mlx5_flow_mreg_copy_resource {
 	 *  - Key is 32/64-bit MARK action ID.
 	 *  - MUST be the first entry.
 	 */
-	struct mlx5_hlist_entry hlist_ent;
+	struct mlx5_list_entry hlist_ent;
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
 	uint32_t idx;
@@ -586,7 +586,7 @@ struct mlx5_flow_tbl_tunnel_prm {
 
 /* Table data structure of the hash organization. */
 struct mlx5_flow_tbl_data_entry {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
@@ -926,7 +926,7 @@ struct mlx5_flow_tunnel_hub {
 
 /* convert jump group to flow table ID in tunnel rules */
 struct tunnel_tbl_entry {
-	struct mlx5_hlist_entry hash;
+	struct mlx5_list_entry hash;
 	uint32_t flow_table;
 	uint32_t tunnel_id;
 	uint32_t group;
@@ -1573,110 +1573,105 @@ int mlx5_action_handle_flush(struct rte_eth_dev *dev);
 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id);
 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh);
 
-/* Hash list callbacks for flow tables: */
-struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *entry_ctx);
-int flow_dv_tbl_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tbl_create_cb(void *tool_ctx, void *entry_ctx);
+int flow_dv_tbl_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
+void flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tbl_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *entry_ctx);
+void flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
 struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 		uint32_t table_level, uint8_t egress, uint8_t transfer,
 		bool external, const struct mlx5_flow_tunnel *tunnel,
 		uint32_t group_id, uint8_t dummy,
 		uint32_t table_id, struct rte_flow_error *error);
 
-struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *cb_ctx);
-int flow_dv_tag_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx);
+int flow_dv_tag_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
-
-int flow_dv_modify_match_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry,
-			    uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
-						  uint64_t key, void *ctx);
-void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry);
-
-struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
-						uint64_t key, void *ctx);
-int flow_dv_mreg_match_cb(struct mlx5_hlist *list,
-			  struct mlx5_hlist_entry *entry, uint64_t key,
+void flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tag_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *cb_ctx);
+void flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_modify_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_modify_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_modify_clone_cb(void *tool_ctx,
+						struct mlx5_list_entry *oentry,
+						void *ctx);
+void flow_dv_modify_clone_free_cb(void *tool_ctx,
+				  struct mlx5_list_entry *entry);
+
+struct mlx5_list_entry *flow_dv_mreg_create_cb(void *tool_ctx, void *ctx);
+int flow_dv_mreg_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			  void *cb_ctx);
-void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry);
-
-int flow_dv_encap_decap_match_cb(struct mlx5_hlist *list,
-				 struct mlx5_hlist_entry *entry,
-				 uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-				uint64_t key, void *cb_ctx);
-void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry);
-
-int flow_dv_matcher_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *ctx);
-struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
+void flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_mreg_clone_cb(void *tool_ctx,
+					      struct mlx5_list_entry *entry,
+					      void *ctx);
+void flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
 
-int flow_dv_port_id_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused,
+int flow_dv_encap_decap_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 				 void *cb_ctx);
-void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused);
-int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
-						 struct mlx5_list_entry *entry,
-						 void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_sample_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
+struct mlx5_list_entry *flow_dv_encap_decap_create_cb(void *tool_ctx,
+						      void *cb_ctx);
+void flow_dv_encap_decap_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_encap_decap_clone_cb(void *tool_ctx,
 						  struct mlx5_list_entry *entry,
 						  void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+void flow_dv_encap_decap_clone_free_cb(void *tool_ctx,
+				       struct mlx5_list_entry *entry);
+
+int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_matcher_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(void *tool_ctx,
+				     struct mlx5_list_entry *entry __rte_unused,
+				     void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry __rte_unused);
+
+int flow_dv_push_vlan_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			       void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(void *tool_ctx,
+						    void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(void *tool_ctx,
+				     struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_sample_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_dest_array_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+
+int flow_dv_dest_array_match_cb(void *tool_ctx,	struct mlx5_list_entry *entry,
+				void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(void *tool_ctx,
+						     void *cb_ctx);
+void flow_dv_dest_array_remove_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(void *tool_ctx,
+				      struct mlx5_list_entry *entry);
+
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 4aafe87524..1dfd5db361 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3580,25 +3580,9 @@ flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
 	return 0;
 }
 
-/**
- * Match encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
 int
-flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
-			     struct mlx5_hlist_entry *entry,
-			     uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3617,25 +3601,10 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 	return -1;
 }
 
-/**
- * Allocate encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
-struct mlx5_hlist_entry *
-flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-			      uint64_t key __rte_unused,
-			      void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3673,6 +3642,38 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			     void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	uint32_t idx;
+
+	cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+					   &idx);
+	if (!cache_resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(cache_resource, oentry, sizeof(*cache_resource));
+	cache_resource->idx = idx;
+	return &cache_resource->entry;
+}
+
+void
+flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *res =
+				       container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
+}
+
 /**
  * Find existing encap/decap resource or create and register a new one.
  *
@@ -3697,7 +3698,7 @@ flow_dv_encap_decap_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	union {
 		struct {
 			uint32_t ft_type:8;
@@ -3774,23 +3775,21 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->port_id != res->port_id;
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
@@ -3821,11 +3820,11 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_clone_cb(void *tool_ctx,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
@@ -3843,12 +3842,11 @@ flow_dv_port_id_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
@@ -3893,23 +3891,21 @@ flow_dv_port_id_action_resource_register
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
+			   struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
@@ -3946,11 +3942,11 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_clone_cb(void *tool_ctx,
+			   struct mlx5_list_entry *entry __rte_unused,
+			   void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx;
@@ -3968,12 +3964,11 @@ flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
@@ -5294,30 +5289,14 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
 	return ret;
 }
 
-/**
- * Match modify-header resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx
- *   Pointer to new modify-header resource.
- *
- * @return
- *   0 on matching, non-zero otherwise.
- */
 int
-flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
-			struct mlx5_hlist_entry *entry,
-			uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	struct mlx5_flow_dv_modify_hdr_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
 
 	key_len += ref->actions_num * sizeof(ref->actions[0]);
@@ -5325,11 +5304,10 @@ flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
-			 void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
@@ -5368,6 +5346,33 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
 	return &entry->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *oentry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *entry;
+	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+
+	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	return &entry->entry;
+}
+
+void
+flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 /**
  * Validate the sample action.
  *
@@ -5639,7 +5644,7 @@ flow_dv_modify_hdr_resource_register
 	uint32_t key_len = sizeof(*resource) -
 			   offsetof(typeof(*resource), ft_type) +
 			   resource->actions_num * sizeof(resource->actions[0]);
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
 		.data = resource,
@@ -9915,7 +9920,7 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 }
 
 static struct mlx5_list_entry *
-flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -9938,22 +9943,22 @@ flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 static void
-flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
-	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data;
+	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
 	struct rte_flow_error *error = ctx->error;
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
 	struct mlx5_flow_tbl_resource *tbl;
 	void *domain;
 	uint32_t idx = 0;
@@ -10010,7 +10015,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
 					      flow_dv_matcher_remove_cb,
@@ -10030,13 +10035,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 }
 
 int
-flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key64,
-		     void *cb_ctx __rte_unused)
+flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
 		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
 
 	return tbl_data->level != key.level ||
 	       tbl_data->id != key.id ||
@@ -10045,6 +10050,39 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
 	       tbl_data->is_egress != !!key.is_egress;
 }
 
+struct mlx5_list_entry *
+flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data;
+	struct rte_flow_error *error = ctx->error;
+	uint32_t idx = 0;
+
+	tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
+	if (!tbl_data) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate flow table data entry");
+		return NULL;
+	}
+	memcpy(tbl_data, oentry, sizeof(*tbl_data));
+	tbl_data->idx = idx;
+	return &tbl_data->entry;
+}
+
+void
+flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data =
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
+}
+
 /**
  * Get a flow table.
  *
@@ -10095,9 +10133,10 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = &tt_prm,
+		.data = &table_key.v64,
+		.data2 = &tt_prm,
 	};
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 
 	entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
@@ -10116,12 +10155,11 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
-		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
 
 	MLX5_ASSERT(entry && sh);
 	if (tbl_data->jump.action)
@@ -10129,7 +10167,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 	if (tbl_data->tbl.obj)
 		mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
 	if (tbl_data->tunnel_offload && tbl_data->external) {
-		struct mlx5_hlist_entry *he;
+		struct mlx5_list_entry *he;
 		struct mlx5_hlist *tunnel_grp_hash;
 		struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
 		union tunnel_tbl_key tunnel_key = {
@@ -10138,11 +10176,14 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			.group = tbl_data->group_id
 		};
 		uint32_t table_level = tbl_data->level;
+		struct mlx5_flow_cb_ctx ctx = {
+			.data = (void *)&tunnel_key.val,
+		};
 
 		tunnel_grp_hash = tbl_data->tunnel ?
 					tbl_data->tunnel->groups :
 					thub->groups;
-		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
+		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
 		if (he)
 			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
@@ -10181,7 +10222,7 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10196,11 +10237,9 @@ flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_matcher_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
 	struct mlx5_flow_dv_matcher *resource;
@@ -10297,29 +10336,29 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 	return 0;
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
+struct mlx5_list_entry *
+flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct rte_flow_error *error = ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_tag_resource *entry;
 	uint32_t idx = 0;
 	int ret;
 
 	entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
 	if (!entry) {
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
 	entry->idx = idx;
-	entry->tag_id = key;
-	ret = mlx5_flow_os_create_flow_action_tag(key,
+	entry->tag_id = *(uint32_t *)(ctx->data);
+	ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
 						  &entry->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create action");
 		return NULL;
@@ -10328,14 +10367,45 @@ flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
 }
 
 int
-flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key,
-		     void *cb_ctx __rte_unused)
+flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_tag_resource *tag =
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+
+	return *(uint32_t *)(ctx->data) != tag->tag_id;
+}
+
+struct mlx5_list_entry *
+flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		     void *cb_ctx)
 {
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_tag_resource *entry;
+	uint32_t idx = 0;
+
+	entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate tag resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry));
+	entry->idx = idx;
+	return &entry->entry;
+}
+
+void
+flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
-	return key != tag->tag_id;
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
 }
 
 /**
@@ -10362,9 +10432,13 @@ flow_dv_tag_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_dv_tag_resource *resource;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_cb_ctx ctx = {
+					.error = error,
+					.data = &tag_be24,
+					};
 
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
+	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
@@ -10376,12 +10450,11 @@ flow_dv_tag_resource_register
 }
 
 void
-flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
 	MLX5_ASSERT(tag && sh && tag->action);
 	claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
@@ -10696,7 +10769,7 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
 			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10725,9 +10798,7 @@ flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -10814,7 +10885,7 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
@@ -10840,16 +10911,15 @@ flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			resource->idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
 }
 
 /**
@@ -10892,14 +10962,14 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
 			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_dv_dest_array_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
 	if (ctx_resource->num_of_dest == resource->num_of_dest &&
@@ -10921,9 +10991,7 @@ flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11028,9 +11096,9 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
+			    struct mlx5_list_entry *entry __rte_unused,
+			    void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11056,8 +11124,8 @@ flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
+				 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
 			container_of(entry, typeof(*resource), entry);
@@ -13531,7 +13599,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
 			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_matcher *resource = container_of(entry,
@@ -13568,19 +13636,10 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Release encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- */
 void
-flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry)
+flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
 				       container_of(entry, typeof(*res), entry);
 
@@ -13640,8 +13699,8 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused,
-			 struct mlx5_hlist_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
@@ -13673,10 +13732,9 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
 				  container_of(entry, typeof(*resource), entry);
 
@@ -13730,10 +13788,9 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
 			container_of(entry, typeof(*resource), entry);
 
@@ -13802,7 +13859,7 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
@@ -13850,7 +13907,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 5450ddd388..3f2b99fb65 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index aa9e973d10..7893b3edd4 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,25 +2093,10 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Match an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   0 if match, none zero if not match.
- */
 int
-mlx5_hrxq_match_cb(struct mlx5_list *list,
-		   struct mlx5_list_entry *entry,
-		   void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2251,10 +2236,9 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
 	__mlx5_hrxq_remove(dev, hrxq);
@@ -2305,25 +2289,10 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
 	return NULL;
 }
 
-/**
- * Create an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   queue entry on success, NULL otherwise.
- */
 struct mlx5_list_entry *
-mlx5_hrxq_create_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry __rte_unused,
-		    void *cb_ctx)
+mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq;
@@ -2333,11 +2302,10 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-mlx5_hrxq_clone_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry,
+mlx5_hrxq_clone_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		    void *cb_ctx __rte_unused)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	uint32_t hrxq_idx = 0;
@@ -2351,10 +2319,9 @@ mlx5_hrxq_clone_cb(struct mlx5_list *list,
 }
 
 void
-mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index a4526444f9..94abe79860 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -8,257 +8,6 @@
 
 #include "mlx5_utils.h"
 
-
-/********************* mlx5 list ************************/
-
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
-{
-	struct mlx5_list *list;
-	int i;
-
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return NULL;
-	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
-	return list;
-}
-
-static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
-{
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
-	uint32_t ret;
-
-	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
-			if (reuse) {
-				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_RELAXED) - 1;
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
-					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
-				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_RELAXED);
-			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
-				return entry;
-			if (reuse && ret == 0)
-				entry->ref_cnt--; /* Invalid entry. */
-		}
-		entry = LIST_NEXT(entry, next);
-	}
-	return NULL;
-}
-
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry = NULL;
-	int i;
-
-	rte_rwlock_read_lock(&list->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
-		if (entry)
-			break;
-	}
-	rte_rwlock_read_unlock(&list->lock);
-	return entry;
-}
-
-static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
-		       struct mlx5_list_entry *gentry, void *ctx)
-{
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
-
-	if (unlikely(!lentry))
-		return NULL;
-	lentry->ref_cnt = 1u;
-	lentry->gentry = gentry;
-	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
-	return lentry;
-}
-
-static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
-{
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
-	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
-	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
-					       __ATOMIC_RELAXED);
-
-	while (inv_cnt != 0 && entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
-			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
-			inv_cnt--;
-		}
-		entry = nentry;
-	}
-}
-
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry, *local_entry;
-	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
-	MLX5_ASSERT(list);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
-	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
-	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
-	if (local_entry)
-		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
-		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
-	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
-	if (unlikely(!entry))
-		return NULL;
-	local_entry = list->cb_clone(list, entry, ctx);
-	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
-		return NULL;
-	}
-	entry->ref_cnt = 1u;
-	local_entry->ref_cnt = 1u;
-	local_entry->gentry = entry;
-	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
-	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
-							       RTE_MAX_LCORE,
-							       ctx, true);
-
-		if (unlikely(oentry)) {
-			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
-		}
-	}
-	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
-		(void *)entry, entry->ref_cnt);
-	return local_entry;
-}
-
-int
-mlx5_list_unregister(struct mlx5_list *list,
-		      struct mlx5_list_entry *entry)
-{
-	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
-
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
-	if (entry->lcore_idx == (uint32_t)lcore_idx) {
-		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
-	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
-				   __ATOMIC_RELAXED);
-	} else {
-		return 0;
-	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	rte_rwlock_write_lock(&list->lock);
-	if (likely(gentry->ref_cnt == 0)) {
-		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
-		return 0;
-	}
-	rte_rwlock_write_unlock(&list->lock);
-	return 1;
-}
-
-void
-mlx5_list_destroy(struct mlx5_list *list)
-{
-	struct mlx5_list_entry *entry;
-	int i;
-
-	MLX5_ASSERT(list);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
-			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
-					(void *)entry);
-			} else {
-				list->cb_clone_free(list, entry);
-			}
-		}
-	}
-	mlx5_free(list);
-}
-
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list)
-{
-	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
-}
-
 /********************* Indexed pool **********************/
 
 static inline void
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 0bf2f5f5ca..7d9b64c877 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -296,203 +296,6 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ mlx5 list *****************************/
-
-/** Maximum size of string for naming. */
-#define MLX5_NAME_SIZE			32
-
-struct mlx5_list;
-
-/**
- * Structure of the entry in the mlx5 list, user should define its own struct
- * that contains this in order to store the data.
- */
-struct mlx5_list_entry {
-	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* 0 means, entry is invalid. */
-	uint32_t lcore_idx;
-	struct mlx5_list_entry *gentry;
-};
-
-struct mlx5_list_cache {
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
-	uint32_t inv_cnt; /* Invalid entries counter. */
-} __rte_cache_aligned;
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
-				   struct mlx5_list_entry *entry, void *ctx);
-
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
-
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined mlx5 list entry creation.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer of entry on success, NULL otherwise.
- */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
-
-/**
- * Linked mlx5 list structure.
- *
- * Entry in mlx5 list could be reused if entry already exists,
- * reference count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- *
- * Linked list is designed for limited number of entries,
- * read mostly, less modification.
- *
- * For huge amount of entries, please consider hash list.
- *
- */
-struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
-	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-};
-
-/**
- * Create a mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param name
- *   Name of the mlx5 list.
- * @param ctx
- *   Pointer to the list context data.
- * @param cb_create
- *   Callback function for entry create.
- * @param cb_match
- *   Callback function for entry match.
- * @param cb_remove
- *   Callback function for entry remove.
- * @return
- *   List pointer on success, otherwise NULL.
- */
-struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
-				   mlx5_list_create_cb cb_create,
-				   mlx5_list_match_cb cb_match,
-				   mlx5_list_remove_cb cb_remove,
-				   mlx5_list_clone_cb cb_clone,
-				   mlx5_list_clone_free_cb cb_clone_free);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param list
- *   Pointer to the mlx5 list.
- * @param ctx
- *   Common context parameter used by entry callback function.
- *
- * @return
- *   Pointer of the list entry if found, NULL otherwise.
- */
-struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
-					   void *ctx);
-
-/**
- * Reuse or create an entry to the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param ctx
- *   Common context parameter used by callback function.
- *
- * @return
- *   registered entry on success, NULL otherwise
- */
-struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
-					     void *ctx);
-
-/**
- * Remove an entry from the mlx5 list.
- *
- * User should guarantee the validity of the entry.
- *
- * @param list
- *   Pointer to the hast list.
- * @param entry
- *   Entry to be removed from the mlx5 list table.
- * @return
- *   0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_list_unregister(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry);
-
-/**
- * Destroy the mlx5 list.
- *
- * @param list
- *   Pointer to the mlx5 list.
- */
-void mlx5_list_destroy(struct mlx5_list *list);
-
-/**
- * Get entry number from the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list.
- * @return
- *   mlx5 list entry number.
- */
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list);
-
 /********************************* indexed pool *************************/
 
 /**
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 97a8f04e39..af2f684648 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -608,7 +608,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v2 13/22] net/mlx5: move modify header allocator to ipool

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Modify header actions are allocated by mlx5_malloc which has a big
overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has
its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in
the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has
better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header
handler.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0e80408511..713accf675 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -801,6 +801,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -814,6 +815,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 896c0c2da1..5774f63244 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1123,6 +1136,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ab4e8c5c4f..4552aaa803 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -504,23 +504,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 1dfd5db361..08704d892a 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5304,6 +5304,45 @@ flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
+static struct mlx5_indexed_pool *
+flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
+{
+	struct mlx5_indexed_pool *ipool = __atomic_load_n
+				     (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
+
+	if (!ipool) {
+		struct mlx5_indexed_pool *expected = NULL;
+		struct mlx5_indexed_pool_config cfg =
+		    (struct mlx5_indexed_pool_config) {
+		       .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
+								   (index + 1) *
+					   sizeof(struct mlx5_modification_cmd),
+		       .trunk_size = 64,
+		       .grow_trunk = 3,
+		       .grow_shift = 2,
+		       .need_lock = 1,
+		       .release_mem_en = 1,
+		       .malloc = mlx5_malloc,
+		       .free = mlx5_free,
+		       .type = "mlx5_modify_action_resource",
+		};
+
+		cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
+		ipool = mlx5_ipool_create(&cfg);
+		if (!ipool)
+			return NULL;
+		if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
+						 &expected, ipool, false,
+						 __ATOMIC_SEQ_CST,
+						 __ATOMIC_SEQ_CST)) {
+			mlx5_ipool_destroy(ipool);
+			ipool = __atomic_load_n(&sh->mdh_ipools[index],
+						__ATOMIC_SEQ_CST);
+		}
+	}
+	return ipool;
+}
+
 struct mlx5_list_entry *
 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
@@ -5312,12 +5351,20 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
+							  ref->actions_num - 1);
 	int ret;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
+	uint32_t idx;
 
-	entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
-			    SOCKET_ID_ANY);
+	if (unlikely(!ipool)) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL, "cannot allocate modify ipool");
+		return NULL;
+	}
+	entry = mlx5_ipool_zmalloc(ipool, &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5337,25 +5384,29 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 					(sh->ctx, ns, entry,
 					 data_len, &entry->action);
 	if (ret) {
-		mlx5_free(entry);
+		mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create modification action");
 		return NULL;
 	}
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 struct mlx5_list_entry *
-flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
-			struct mlx5_list_entry *oentry, void *cb_ctx)
+flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			void *cb_ctx)
 {
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+	uint32_t idx;
 
-	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
+				  &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5363,14 +5414,18 @@ flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
 		return NULL;
 	}
 	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 void
-flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	mlx5_free(entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *res =
+		container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
@@ -13699,14 +13754,14 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
-			 struct mlx5_list_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
-	mlx5_free(entry);
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v2 14/22] net/mlx5: adjust the hash bucket size

[Suanming Mou]

With the new per core optimization to the list, the hash bucket size
can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index cf573a9a4d..a82dc4db00 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 713accf675..8fb7f4442d 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -373,7 +373,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 906aa43c5a..ca67ce8213 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index e6b71a87a0..cb8161f668 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -8625,7 +8625,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8734,8 +8734,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-- 
2.25.1

[dpdk-dev] [PATCH v2 15/22] common/mlx5: allocate cache list memory individually

[Suanming Mou]

Currently, the list's local cache instance memory is allocated with
the list. As the local cache instance array size is RTE_MAX_LCORE,
most of the cases the system will only have very limited cores.
allocate the instance memory individually per core will be more
economic to the memory.

This commit changes the instance array to pointer array, allocate
the local cache memory only when the core is to be used.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 4e385c616a..f75b1cb0da 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 79d0bc1a50..42ca3cbc1e 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-- 
2.25.1

[dpdk-dev] [PATCH v2 16/22] net/mlx5: enable index pool per-core cache

[Suanming Mou]

This commit enables the tag and header modify action index pool
per-core cache in non-reclaim memory mode.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 8fb7f4442d..bf1463c289 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -214,7 +214,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1128,6 +1129,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 5774f63244..516f3ffae5 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1093,6 +1093,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 08704d892a..f79c60e489 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5321,7 +5321,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
-- 
2.25.1

[dpdk-dev] [PATCH v2 17/22] net/mlx5: optimize hash list table allocate on demand

[Suanming Mou]

Currently, all the hash list tables are allocated during start up.
Since different applications may only use dedicated limited actions,
optimized the hash list table allocate on demand will save initial
memory.

This commit optimizes hash list table allocate on demand.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index a82dc4db00..75324e35d8 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index ca67ce8213..fe86bb40d3 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index f79c60e489..fe610594c5 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -310,6 +310,41 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 	}
 }
 
+static inline struct mlx5_hlist *
+flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
+		     const char *name, uint32_t size, bool direct_key,
+		     bool lcores_share, void *ctx,
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove,
+		     mlx5_list_clone_cb cb_clone,
+		     mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_hlist *hl;
+	struct mlx5_hlist *expected = NULL;
+	char s[MLX5_NAME_SIZE];
+
+	hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	if (likely(hl))
+		return hl;
+	snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
+	hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
+			ctx, cb_create, cb_match, cb_remove, cb_clone,
+			cb_clone_free);
+	if (!hl) {
+		DRV_LOG(ERR, "%s hash creation failed", name);
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
+					 __ATOMIC_SEQ_CST,
+					 __ATOMIC_SEQ_CST)) {
+		mlx5_hlist_destroy(hl);
+		hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	}
+	return hl;
+}
+
 /* Update VLAN's VID/PCP based on input rte_flow_action.
  *
  * @param[in] action
@@ -3724,8 +3759,20 @@ flow_dv_encap_decap_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *encaps_decaps;
 	uint64_t key64;
 
+	encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
+				"encaps_decaps",
+				MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
+				true, true, sh,
+				flow_dv_encap_decap_create_cb,
+				flow_dv_encap_decap_match_cb,
+				flow_dv_encap_decap_remove_cb,
+				flow_dv_encap_decap_clone_cb,
+				flow_dv_encap_decap_clone_free_cb);
+	if (unlikely(!encaps_decaps))
+		return -rte_errno;
 	resource->flags = dev_flow->dv.group ? 0 : 1;
 	key64 =  __rte_raw_cksum(&encap_decap_key.v32,
 				 sizeof(encap_decap_key.v32), 0);
@@ -3733,7 +3780,7 @@ flow_dv_encap_decap_resource_register
 	    MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
 	    resource->size)
 		key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
-	entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
+	entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -5705,8 +5752,20 @@ flow_dv_modify_hdr_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *modify_cmds;
 	uint64_t key64;
 
+	modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
+				"hdr_modify",
+				MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
+				true, false, sh,
+				flow_dv_modify_create_cb,
+				flow_dv_modify_match_cb,
+				flow_dv_modify_remove_cb,
+				flow_dv_modify_clone_cb,
+				flow_dv_modify_clone_free_cb);
+	if (unlikely(!modify_cmds))
+		return -rte_errno;
 	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
 								resource->root))
@@ -5714,7 +5773,7 @@ flow_dv_modify_hdr_resource_register
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
 	key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
-	entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
+	entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10493,8 +10552,20 @@ flow_dv_tag_resource_register
 					.error = error,
 					.data = &tag_be24,
 					};
-
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
+	struct mlx5_hlist *tag_table;
+
+	tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
+				      "tags",
+				      MLX5_TAGS_HLIST_ARRAY_SIZE,
+				      false, false, priv->sh,
+				      flow_dv_tag_create_cb,
+				      flow_dv_tag_match_cb,
+				      flow_dv_tag_remove_cb,
+				      flow_dv_tag_clone_cb,
+				      flow_dv_tag_clone_free_cb);
+	if (unlikely(!tag_table))
+		return -rte_errno;
+	entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index af2f684648..0e6f7003b0 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
-- 
2.25.1

[dpdk-dev] [PATCH v2 18/22] common/mlx5: optimize cache list object memory

[Suanming Mou]

Currently, hash list uses the cache list as bucket list. The list
in the buckets have the same name, ctx and callbacks. This wastes
the memory.

This commit abstracts all the name, ctx and callback members in the
list to a constant struct and others to the inconstant struct, uses
the wrapper functions to satisfy both hash list and cache list can
set the list constant and inconstant struct individually.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index f75b1cb0da..858c8d8164 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 42ca3cbc1e..6acd0d3754 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -80,6 +80,32 @@ typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
 typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
 						       void *ctx);
 
+/**
+ * Linked mlx5 list constant object.
+ */
+struct mlx5_list_const {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	void *ctx; /* user objects target to callback. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	/**< entry clone free callback. */
+};
+
+/**
+ * Linked mlx5 list inconstant data.
+ */
+struct mlx5_list_inconst {
+	rte_rwlock_t lock; /* read/write lock. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+};
+
 /**
  * Linked mlx5 list structure.
  *
@@ -96,19 +122,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
  *
  */
 struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	void *ctx; /* user objects target to callback. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-	volatile uint32_t gen_cnt; /* List modification may update it. */
-	volatile uint32_t count; /* number of entries in list. */
-	rte_rwlock_t lock; /* read/write lock. */
+	struct mlx5_list_const l_const;
+	struct mlx5_list_inconst l_inconst;
 };
 
 /**
@@ -208,7 +223,7 @@ mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_list l;
+	struct mlx5_list_inconst l;
 } __rte_cache_aligned;
 
 /**
@@ -220,7 +235,7 @@ struct mlx5_hlist {
 	uint32_t mask; /* A mask for the bucket index range. */
 	uint8_t flags;
 	bool direct_key; /* Whether to use the key directly as hash index. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
+	struct mlx5_list_const l_const; /* List constant data. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v2 19/22] net/mlx5: change memory release configuration

[Suanming Mou]

This commit changes the index pool memory release configuration
to 0 when memory reclaim mode is not required.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index bf1463c289..6b7225e55d 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -797,6 +797,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
-- 
2.25.1

[dpdk-dev] [PATCH v2 20/22] net/mlx5: support index pool none local core operations

[Suanming Mou]

This commit supports the index pool none local core operations with
an extra cache.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 94abe79860..c34d6d62a8 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -114,6 +114,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->nlcore_lock);
 	return pool;
 }
 
@@ -354,20 +355,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -378,15 +373,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -399,29 +406,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -464,6 +482,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -643,7 +676,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -712,7 +745,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 7d9b64c877..060c52f022 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t nlcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
-- 
2.25.1

[dpdk-dev] [PATCH v2 21/22] net/mlx5: support list none local core operations

[Suanming Mou]

This commit supports the list none local core operations with
an extra sub-list.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 858c8d8164..d58d0d08ab 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.nlcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.nlcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.nlcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.nlcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.nlcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.nlcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.nlcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.nlcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.nlcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.nlcore_lock);
+	return ret;
 }
 
 void
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 6acd0d3754..66ce27464c 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t nlcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v2 22/22] net/mlx5: optimize Rx queue match

[Suanming Mou]

As hrxq struct has the indirect table pointer, while matching the
hrxq, better to use the hrxq indirect table instead of searching
from the list.

This commit optimizes the hrxq indirect table matching.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 7893b3edd4..23685d7654 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v3 00/22] net/mlx5: insertion rate optimization

[Suanming Mou]

This patch series optimize the flow insertion rate with adding
local cache to index pool and list.

For object which wants efficient index allocate and free, local
cache will be very helpful.

For index pool, two level cache is added, one as local and another
as global. The global cache is able to save all the allocated
index. That means all the allocated index will not be freed. Once
the local cache is full, the extra index will be flushed to the
global cache. Once local cache is empty, first try to fetch more
index from global, if global is still empty, allocate new trunk
and more index.

For list, sub local core list is introduced. The allocated objects
will be added and released only from local list without any locks.
Only the objects need to be shared will be synced from global list.

---

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
  net/mlx5: optimize modify header action memory
  net/mlx5: remove cache term from the list utility
  net/mlx5: add per lcore cache to the list utility
  net/mlx5: minimize list critical sections
  net/mlx5: manage list cache entries release
  net/mlx5: relax the list utility atomic operations
  net/mlx5: allocate list memory by the create API
  common/mlx5: add per-lcore cache to hash list utility
  net/mlx5: move modify header allocator to ipool

Suanming Mou (13):
  net/mlx5: allow limiting the index pool maximum index
  net/mlx5: add indexed pool local cache
  net/mlx5: add index pool foreach define
  net/mlx5: replace flow list with index pool
  net/mlx5: adjust the hash bucket size
  common/mlx5: allocate cache list memory individually
  net/mlx5: enable index pool per-core cache
  net/mlx5: optimize hash list table allocate on demand
  common/mlx5: optimize cache list object memory
  net/mlx5: change memory release configuration
  net/mlx5: support index pool none local core operations
  net/mlx5: support list none local core operations
  net/mlx5: optimize Rx queue match

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    6 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  289 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  305 +++---
 drivers/net/mlx5/mlx5_flow.h            |  210 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1203 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  617 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2531 insertions(+), 1411 deletions(-)

-- 
2.25.1

[dpdk-dev] [PATCH v3 01/22] net/mlx5: allow limiting the index pool maximum index

[Suanming Mou]

Some ipool instances in the driver are used as ID\index allocator and
added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value.
The ipool will ensure that no index bigger than the maximum value is
provided.

Use this configuration in ID allocator cases instead of the current
logics. This patch add the maximum ID configurable for the index pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 18fe23e4fb..bf2b2ebc72 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index b54517c6df..15870e14c2 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
-- 
2.25.1

[dpdk-dev] [PATCH v3 02/22] net/mlx5: add indexed pool local cache

[Suanming Mou]

For object which wants efficient index allocate and free, local
cache will be very helpful.

Two level cache is introduced to allocate and free the index more
efficient. One as local and the other as global. The global cache
is able to save all the allocated index. That means all the allocated
index will not be freed. Once the local cache is full, the extra
index will be flushed to the global cache. Once local cache is empty,
first try to fetch more index from global, if global is still empty,
allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index bf2b2ebc72..215024632d 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 15870e14c2..0469062695 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -209,6 +209,11 @@ struct mlx5_indexed_pool_config {
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
 	uint32_t max_idx; /* The maximum index can be allocated. */
+	uint32_t per_core_cache;
+	/*
+	 * Cache entry number per core for performance. Should not be
+	 * set with release_mem_en.
+	 */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
@@ -225,14 +230,39 @@ struct mlx5_indexed_trunk {
 	uint8_t data[] __rte_cache_aligned; /* Entry data start. */
 };
 
+struct mlx5_indexed_cache {
+	struct mlx5_indexed_trunk **trunks;
+	volatile uint32_t n_trunk_valid; /* Trunks allocated. */
+	uint32_t n_trunk; /* Trunk pointer array size. */
+	uint32_t ref_cnt;
+	uint32_t len;
+	uint32_t idx[];
+};
+
+struct mlx5_ipool_per_lcore {
+	struct mlx5_indexed_cache *lc;
+	uint32_t len; /**< Current cache count. */
+	uint32_t idx[]; /**< Cache objects. */
+};
+
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
-	rte_spinlock_t lock; /* Pool lock for multiple thread usage. */
-	uint32_t n_trunk_valid; /* Trunks allocated. */
-	uint32_t n_trunk; /* Trunk pointer array size. */
+	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
 	/* Dim of trunk pointer array. */
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t free_list; /* Index to first free trunk. */
+	union {
+		struct {
+			uint32_t n_trunk_valid; /* Trunks allocated. */
+			uint32_t n_trunk; /* Trunk pointer array size. */
+			struct mlx5_indexed_trunk **trunks;
+			uint32_t free_list; /* Index to first free trunk. */
+		};
+		struct {
+			struct mlx5_indexed_cache *gc;
+			/* Global cache. */
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			/* Local cache. */
+		};
+	};
 #ifdef POOL_DEBUG
 	uint32_t n_entry;
 	uint32_t trunk_new;
@@ -542,6 +572,30 @@ int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool);
  */
 void mlx5_ipool_dump(struct mlx5_indexed_pool *pool);
 
+/**
+ * This function flushes all the cache index back to pool trunk.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ *
+ */
+
+void mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool);
+
+/**
+ * This function gets the available entry from pos.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ * @param pos
+ *   Pointer to the index position start from.
+ *
+ * @return
+ *  - Pointer to the next available entry.
+ *
+ */
+void *mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos);
+
 /**
  * This function allocates new empty Three-level table.
  *
-- 
2.25.1

[dpdk-dev] [PATCH v3 03/22] net/mlx5: add index pool foreach define

[Suanming Mou]

In some cases, application may want to know all the allocated
index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach
operation.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 86 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 ++++
 2 files changed, 94 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 215024632d..0ed279e162 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -839,6 +839,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 0469062695..737dd7052d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
-- 
2.25.1

[dpdk-dev] [PATCH v3 04/22] net/mlx5: replace flow list with index pool

[Suanming Mou]

The flow list is used to save the create flows and to be used only
when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to
flush all the allocated flows.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 +++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 149 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 126 insertions(+), 110 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 92b3009786..31cc8d9eb8 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -823,6 +861,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1566,7 +1605,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1600,6 +1638,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cf1815cb74..fcfc3dcdca 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -322,7 +322,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -792,8 +793,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1528,7 +1531,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 1b2dc8f815..380d35d420 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1344,7 +1352,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1596,7 +1605,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 3b7c94d92f..450a84a6c5 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3095,31 +3095,6 @@ mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
 					 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
 }
 
-/**
- * Release resource related QUEUE/RSS action split.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Flow to release id's from.
- */
-static void
-flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
-			     struct rte_flow *flow)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t handle_idx;
-	struct mlx5_flow_handle *dev_handle;
-
-	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
-		       handle_idx, dev_handle, next)
-		if (dev_handle->split_flow_id &&
-		    !dev_handle->is_meter_flow_id)
-			mlx5_ipool_free(priv->sh->ipool
-					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
-					dev_handle->split_flow_id);
-}
-
 static int
 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
 		   const struct rte_flow_attr *attr __rte_unused,
@@ -3415,7 +3390,6 @@ flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 	const struct mlx5_flow_driver_ops *fops;
 	enum mlx5_flow_drv_type type = flow->drv_type;
 
-	flow_mreg_split_qrss_release(dev, flow);
 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
 	fops = flow_get_drv_ops(type);
 	fops->destroy(dev, flow);
@@ -3998,14 +3972,14 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
 
 /* Declare flow create/destroy prototype in advance. */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action actions[],
 		 bool external, struct rte_flow_error *error);
 
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
@@ -4127,8 +4101,8 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	 * be applied, removed, deleted in ardbitrary order
 	 * by list traversing.
 	 */
-	mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
-					 actions, false, error);
+	mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
+					&attr, items, actions, false, error);
 	if (!mcp_res->rix_flow) {
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
 		return NULL;
@@ -4190,7 +4164,7 @@ flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
-	flow_list_destroy(dev, NULL, mcp_res->rix_flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
 }
 
@@ -6093,7 +6067,7 @@ flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
  *   A flow index on success, 0 otherwise and rte_errno is set.
  */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action original_actions[],
@@ -6161,7 +6135,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 				external, hairpin_flow, error);
 	if (ret < 0)
 		goto error_before_hairpin_split;
-	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
+	flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
 	if (!flow) {
 		rte_errno = ENOMEM;
 		goto error_before_hairpin_split;
@@ -6291,12 +6265,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 		if (ret < 0)
 			goto error;
 	}
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
-			     flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
+	flow->type = type;
 	flow_rxq_flags_set(dev, flow);
 	rte_free(translated_actions);
 	tunnel = flow_tunnel_from_rule(wks->flows);
@@ -6318,7 +6287,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_ipool_get
 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
 			rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
+	mlx5_ipool_free(priv->flows[type], idx);
 	rte_errno = ret; /* Restore rte_errno. */
 	ret = rte_errno;
 	rte_errno = ret;
@@ -6370,10 +6339,9 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
 			.type = RTE_FLOW_ACTION_TYPE_END,
 		},
 	};
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow_error error;
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
 						   &attr, &pattern,
 						   actions, false, &error);
 }
@@ -6425,8 +6393,6 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
 	/*
 	 * If the device is not started yet, it is not allowed to created a
 	 * flow from application. PMD default flows and traffic control flows
@@ -6442,8 +6408,9 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		return NULL;
 	}
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
-				  attr, items, actions, true, error);
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
+						   attr, items, actions,
+						   true, error);
 }
 
 /**
@@ -6451,24 +6418,19 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list. If this parameter NULL,
- *   there is no flow removal from the list. Be noted that as
- *   flow is add to the indexed list, memory of the indexed
- *   list points to maybe changed as flow destroyed.
  * @param[in] flow_idx
  *   Index of flow to destroy.
  */
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW], flow_idx);
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
 
 	if (!flow)
 		return;
+	MLX5_ASSERT(flow->type == type);
 	/*
 	 * Update RX queue flags only if port is started, otherwise it is
 	 * already clean.
@@ -6476,12 +6438,6 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
 	flow_drv_destroy(dev, flow);
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
-			     flow_idx, flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
 	if (flow->tunnel) {
 		struct mlx5_flow_tunnel *tunnel;
 
@@ -6491,7 +6447,7 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
 	flow_mreg_del_copy_action(dev, flow);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
+	mlx5_ipool_free(priv->flows[type], flow_idx);
 }
 
 /**
@@ -6499,18 +6455,21 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list.
+ * @param type
+ *   Flow type to be flushed.
  * @param active
  *   If flushing is called avtively.
  */
 void
-mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
+mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+		     bool active)
 {
-	uint32_t num_flushed = 0;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t num_flushed = 0, fidx = 1;
+	struct rte_flow *flow;
 
-	while (*list) {
-		flow_list_destroy(dev, list, *list);
+	MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
+		flow_list_destroy(dev, type, fidx);
 		num_flushed++;
 	}
 	if (active) {
@@ -6682,18 +6641,19 @@ mlx5_flow_pop_thread_workspace(void)
  * @return the number of flows not released.
  */
 int
-mlx5_flow_verify(struct rte_eth_dev *dev)
+mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
-	uint32_t idx;
-	int ret = 0;
+	uint32_t idx = 0;
+	int ret = 0, i;
 
-	ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
-		      flow, next) {
-		DRV_LOG(DEBUG, "port %u flow %p still referenced",
-			dev->data->port_id, (void *)flow);
-		++ret;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
+			DRV_LOG(DEBUG, "port %u flow %p still referenced",
+				dev->data->port_id, (void *)flow);
+			ret++;
+		}
 	}
 	return ret;
 }
@@ -6713,7 +6673,6 @@ int
 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 			    uint32_t queue)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.egress = 1,
 		.priority = 0,
@@ -6746,8 +6705,8 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 	actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
 	actions[0].conf = &jump;
 	actions[1].type = RTE_FLOW_ACTION_TYPE_END;
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx) {
 		DRV_LOG(DEBUG,
 			"Failed to create ctrl flow: rte_errno(%d),"
@@ -6836,8 +6795,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 		action_rss.types = 0;
 	for (i = 0; i != priv->reta_idx_n; ++i)
 		queue[i] = (*priv->reta_idx)[i];
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx)
 		return -rte_errno;
 	return 0;
@@ -6878,7 +6837,6 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev,
 int
 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	/*
 	 * The LACP matching is done by only using ether type since using
 	 * a multicast dst mac causes kernel to give low priority to this flow.
@@ -6912,8 +6870,9 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 		},
 	};
 	struct rte_flow_error error;
-	uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+					&attr, items, actions,
+					false, &error);
 
 	if (!flow_idx)
 		return -rte_errno;
@@ -6931,9 +6890,8 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
 		  struct rte_flow *flow,
 		  struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+				(uintptr_t)(void *)flow);
 	return 0;
 }
 
@@ -6947,9 +6905,7 @@ int
 mlx5_flow_flush(struct rte_eth_dev *dev,
 		struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
 	return 0;
 }
 
@@ -7000,8 +6956,7 @@ flow_drv_query(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct mlx5_flow_driver_ops *fops;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW],
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 					       flow_idx);
 	enum mlx5_flow_drv_type ftype;
 
@@ -7867,14 +7822,14 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		if (!config->dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
-		flow_idx = flow_list_create(dev, NULL, &attr, items,
-					    actions, false, &error);
-		flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
+					items, actions, false, &error);
+		flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 				      flow_idx);
 		if (!flow)
 			continue;
 		config->flow_mreg_c[n++] = idx;
-		flow_list_destroy(dev, NULL, flow_idx);
+		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
 	}
 	for (; n < MLX5_MREG_C_NUM; ++n)
 		config->flow_mreg_c[n] = REG_NON;
@@ -7918,8 +7873,8 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
 					sh->rx_domain,
 					sh->tx_domain, file);
 	/* dump one */
-	flow = mlx5_ipool_get(priv->sh->ipool
-			[MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
+	flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
+			(uintptr_t)(void *)flow_idx);
 	if (!flow)
 		return -ENOENT;
 
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2f2aa962f9..d9b6acaafd 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -997,9 +997,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index a04a3c2bb8..e898c571da 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13844,6 +13844,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ae7fcca229..7cb8920d6b 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 3fe3f55f49..7d15c998bb 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	TAILQ_INIT(&priv->flow_meter_profiles);
-- 
2.25.1

[dpdk-dev] [PATCH v3 05/22] net/mlx5: optimize modify header action memory

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Define the types of the modify header action fields to be with the
minimum size needed for the optional values range.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_glue.h b/drivers/common/mlx5/linux/mlx5_glue.h
index 840d8cf57f..a186ee577f 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.h
+++ b/drivers/common/mlx5/linux/mlx5_glue.h
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
diff --git a/drivers/net/mlx5/linux/mlx5_flow_os.h b/drivers/net/mlx5/linux/mlx5_flow_os.h
index cee685015b..1926d26410 100644
--- a/drivers/net/mlx5/linux/mlx5_flow_os.h
+++ b/drivers/net/mlx5/linux/mlx5_flow_os.h
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index d9b6acaafd..81c95e0beb 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -523,11 +523,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index e898c571da..a7c1cf05da 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5000,21 +5000,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5582,10 +5582,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
-- 
2.25.1

[dpdk-dev] [PATCH v3 06/22] net/mlx5: remove cache term from the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The internal mlx5 list tool is used mainly when the list objects need to
be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread
cache management.

To prevent confusing, remove the current "cache" term from the API's
names.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 558 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 454 insertions(+), 460 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 31cc8d9eb8..9aa57e38b7 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -500,8 +500,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1702,7 +1702,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1761,7 +1761,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index fcfc3dcdca..9aade013c5 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1611,7 +1611,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 380d35d420..58646da331 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1221,7 +1221,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1369,7 +1369,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 81c95e0beb..4dec703366 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -467,7 +467,7 @@ struct mlx5_flow_dv_match_params {
 
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
-	struct mlx5_cache_entry entry; /**< Pointer to the next element. */
+	struct mlx5_list_entry entry; /**< Pointer to the next element. */
 	struct mlx5_flow_tbl_resource *tbl;
 	/**< Pointer to the table(group) the matcher associated with. */
 	void *matcher_object; /**< Pointer to DV matcher */
@@ -547,7 +547,7 @@ struct mlx5_flow_dv_jump_tbl_resource {
 
 /* Port ID resource structure. */
 struct mlx5_flow_dv_port_id_action_resource {
-	struct mlx5_cache_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Action object. */
 	uint32_t port_id; /**< Port ID value. */
 	uint32_t idx; /**< Indexed pool memory index. */
@@ -555,7 +555,7 @@ struct mlx5_flow_dv_port_id_action_resource {
 
 /* Push VLAN action resource structure */
 struct mlx5_flow_dv_push_vlan_action_resource {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* Cache entry. */
 	void *action; /**< Action object. */
 	uint8_t ft_type; /**< Flow table type, Rx, Tx or FDB. */
 	rte_be32_t vlan_tag; /**< VLAN tag value. */
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_cache_list matchers;
+	struct mlx5_list matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
@@ -631,7 +631,7 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	union {
 		void *verbs_action; /**< Verbs sample action object. */
 		void **sub_actions; /**< Sample sub-action array. */
@@ -653,7 +653,7 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
@@ -1619,43 +1619,45 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
 
-int flow_dv_matcher_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *ctx);
-struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_port_id_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_sample_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
-			      struct mlx5_cache_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
-				struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_dest_array_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry);
+int flow_dv_matcher_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *ctx);
+void flow_dv_matcher_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
+						 struct mlx5_list_entry *entry,
+						 void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index a7c1cf05da..897bcf52a6 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3601,18 +3601,17 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 			     uint64_t key __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
-
-	cache_resource = container_of(entry,
-				      struct mlx5_flow_dv_encap_decap_resource,
-				      entry);
-	if (resource->reformat_type == cache_resource->reformat_type &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->flags == cache_resource->flags &&
-	    resource->size == cache_resource->size &&
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
+
+	resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
+				entry);
+	if (resource->reformat_type == ctx_resource->reformat_type &&
+	    resource->ft_type == ctx_resource->ft_type &&
+	    resource->flags == ctx_resource->flags &&
+	    resource->size == ctx_resource->size &&
 	    !memcmp((const void *)resource->buf,
-		    (const void *)cache_resource->buf,
+		    (const void *)ctx_resource->buf,
 		    resource->size))
 		return 0;
 	return -1;
@@ -3639,31 +3638,30 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 	uint32_t idx;
 	int ret;
 
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
 	/* Register new encap/decap resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-				       &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
-	cache_resource->idx = idx;
-	ret = mlx5_flow_os_create_flow_action_packet_reformat
-					(sh->ctx, domain, cache_resource,
-					 &cache_resource->action);
+	*resource = *ctx_resource;
+	resource->idx = idx;
+	ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
+							      resource,
+							     &resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3672,7 +3670,7 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 		return NULL;
 	}
 
-	return &cache_resource->entry;
+	return &resource->entry;
 }
 
 /**
@@ -3776,8 +3774,8 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
@@ -3787,30 +3785,30 @@ flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->port_id != res->port_id;
 }
 
-struct mlx5_cache_entry *
-flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_port_id_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate port_id action cache memory");
+				   "cannot allocate port_id action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
 							ref->port_id,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3818,8 +3816,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 				   "cannot create action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3827,8 +3825,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3840,30 +3838,30 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_port_id_action_resource_register
 			(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_port_id_action_resource *resource,
+			 struct mlx5_flow_dv_port_id_action_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->dv.port_id_action = cache;
-	dev_flow->handle->rix_port_id_action = cache->idx;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->dv.port_id_action = resource;
+	dev_flow->handle->rix_port_id_action = resource->idx;
 	return 0;
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
@@ -3873,28 +3871,28 @@ flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
-struct mlx5_cache_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	struct mlx5dv_dr_domain *domain;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate push_vlan action cache memory");
+				   "cannot allocate push_vlan action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
@@ -3902,7 +3900,7 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 	else
 		domain = sh->tx_domain;
 	ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3910,8 +3908,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 				   "cannot create push vlan action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3919,8 +3917,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
  *
  * @param [in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3932,25 +3930,25 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_push_vlan_action_resource_register
 		       (struct rte_eth_dev *dev,
-			struct mlx5_flow_dv_push_vlan_action_resource *resource,
+			struct mlx5_flow_dv_push_vlan_action_resource *ref,
 			struct mlx5_flow *dev_flow,
 			struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
+	resource = container_of(entry, typeof(*resource), entry);
 
-	dev_flow->handle->dvh.rix_push_vlan = cache->idx;
-	dev_flow->dv.push_vlan_res = cache;
+	dev_flow->handle->dvh.rix_push_vlan = resource->idx;
+	dev_flow->dv.push_vlan_res = resource;
 	return 0;
 }
 
@@ -9913,13 +9911,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 			return NULL;
 		}
 	}
-	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache",
+	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh,
-			     flow_dv_matcher_create_cb,
-			     flow_dv_matcher_match_cb,
-			     flow_dv_matcher_remove_cb);
+	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+			 flow_dv_matcher_create_cb,
+			 flow_dv_matcher_match_cb,
+			 flow_dv_matcher_remove_cb);
 	return &tbl_data->entry;
 }
 
@@ -10047,7 +10045,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_cache_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(&tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10075,8 +10073,8 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
@@ -10089,15 +10087,15 @@ flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
 		      (const void *)ref->mask.buf, ref->mask.size);
 }
 
-struct mlx5_cache_entry *
-flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_matcher_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5dv_flow_matcher_attr dv_attr = {
 		.type = IBV_FLOW_ATTR_NORMAL,
 		.match_mask = (void *)&ref->mask,
@@ -10106,29 +10104,30 @@ flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
 							    typeof(*tbl), tbl);
 	int ret;
 
-	cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY);
-	if (!cache) {
+	resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
+			       SOCKET_ID_ANY);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	dv_attr.match_criteria_enable =
-		flow_dv_matcher_enable(cache->mask.buf);
+		flow_dv_matcher_enable(resource->mask.buf);
 	dv_attr.priority = ref->priority;
 	if (tbl->is_egress)
 		dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
 	ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
-					       &cache->matcher_object);
+					       &resource->matcher_object);
 	if (ret) {
-		mlx5_free(cache);
+		mlx5_free(resource);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	return &cache->entry;
+	return &resource->entry;
 }
 
 /**
@@ -10157,8 +10156,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 			 uint32_t group_id,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5_flow_tbl_resource *tbl;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 	struct mlx5_flow_cb_ctx ctx = {
@@ -10178,15 +10177,15 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 					  "cannot allocate ref memory");
 	}
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->handle->dvh.matcher = cache;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.matcher = resource;
 	return 0;
 }
 
@@ -10254,15 +10253,15 @@ flow_dv_tag_resource_register
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_tag_resource *cache_resource;
+	struct mlx5_flow_dv_tag_resource *resource;
 	struct mlx5_hlist_entry *entry;
 
 	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
 	if (entry) {
-		cache_resource = container_of
-			(entry, struct mlx5_flow_dv_tag_resource, entry);
-		dev_flow->handle->dvh.rix_tag = cache_resource->idx;
-		dev_flow->dv.tag_resource = cache_resource;
+		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
+					entry);
+		dev_flow->handle->dvh.rix_tag = resource->idx;
+		dev_flow->dv.tag_resource = resource;
 		return 0;
 	}
 	return -rte_errno;
@@ -10589,68 +10588,69 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused,
-			struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-
-	if (resource->ratio == cache_resource->ratio &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->ft_id == cache_resource->ft_id &&
-	    resource->set_action == cache_resource->set_action &&
-	    !memcmp((void *)&resource->sample_act,
-		    (void *)&cache_resource->sample_act,
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+
+	if (ctx_resource->ratio == resource->ratio &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    ctx_resource->ft_id == resource->ft_id &&
+	    ctx_resource->set_action == resource->set_action &&
+	    !memcmp((void *)&ctx_resource->sample_act,
+		    (void *)&resource->sample_act,
 		    sizeof(struct mlx5_flow_sub_actions_list))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
 		flow_dv_sample_sub_actions_release(dev,
-						&resource->sample_idx);
+						   &ctx_resource->sample_idx);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	void **sample_dv_actions = resource->sub_actions;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	void **sample_dv_actions = ctx_resource->sub_actions;
+	struct mlx5_flow_dv_sample_resource *resource;
 	struct mlx5dv_dr_flow_sampler_attr sampler_attr;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	struct mlx5_flow_tbl_resource *tbl;
 	uint32_t idx = 0;
 	const uint32_t next_ft_step = 1;
-	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
+	uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
 	uint8_t is_egress = 0;
 	uint8_t is_transfer = 0;
 	struct rte_flow_error *error = ctx->error;
 
 	/* Register new sample resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	/* Create normal path table level */
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		is_transfer = 1;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
 		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
 					is_egress, is_transfer,
@@ -10663,8 +10663,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 					  "for sample");
 		goto error;
 	}
-	cache_resource->normal_path_tbl = tbl;
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
+	resource->normal_path_tbl = tbl;
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
 		if (!sh->default_miss_action) {
 			rte_flow_error_set(error, ENOMEM,
 						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10673,33 +10673,33 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 						"created");
 			goto error;
 		}
-		sample_dv_actions[resource->sample_act.actions_num++] =
+		sample_dv_actions[ctx_resource->sample_act.actions_num++] =
 						sh->default_miss_action;
 	}
 	/* Create a DR sample action */
-	sampler_attr.sample_ratio = cache_resource->ratio;
+	sampler_attr.sample_ratio = resource->ratio;
 	sampler_attr.default_next_table = tbl->obj;
-	sampler_attr.num_sample_actions = resource->sample_act.actions_num;
+	sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
 	sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
 							&sample_dv_actions[0];
-	sampler_attr.action = cache_resource->set_action;
+	sampler_attr.action = resource->set_action;
 	if (mlx5_os_flow_dr_create_flow_action_sampler
-			(&sampler_attr, &cache_resource->verbs_action)) {
+			(&sampler_attr, &resource->verbs_action)) {
 		rte_flow_error_set(error, ENOMEM,
 					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					NULL, "cannot create sample action");
 		goto error;
 	}
-	cache_resource->idx = idx;
-	cache_resource->dev = dev;
-	return &cache_resource->entry;
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
 error:
-	if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
 		flow_dv_sample_sub_actions_release(dev,
-						   &cache_resource->sample_idx);
-	if (cache_resource->normal_path_tbl)
+						   &resource->sample_idx);
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-				cache_resource->normal_path_tbl);
+				resource->normal_path_tbl);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
 	return NULL;
 
@@ -10710,8 +10710,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to sample resource.
+ * @param[in] ref
+ *   Pointer to sample resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10722,66 +10722,66 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_sample_resource *resource,
+			 struct mlx5_flow_dv_sample_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_sample = cache_resource->idx;
-	dev_flow->dv.sample_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_sample = resource->idx;
+	dev_flow->dv.sample_res = resource;
 	return 0;
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused,
-			    struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
-	if (resource->num_of_dest == cache_resource->num_of_dest &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    !memcmp((void *)cache_resource->sample_act,
-		    (void *)resource->sample_act,
-		   (resource->num_of_dest *
+	if (ctx_resource->num_of_dest == resource->num_of_dest &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    !memcmp((void *)resource->sample_act,
+		    (void *)ctx_resource->sample_act,
+		   (ctx_resource->num_of_dest *
 		   sizeof(struct mlx5_flow_sub_actions_list)))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
-		for (idx = 0; idx < resource->num_of_dest; idx++)
+		for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 			flow_dv_sample_sub_actions_release(dev,
-					&resource->sample_idx[idx]);
+					&ctx_resource->sample_idx[idx]);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
 	struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -10794,23 +10794,23 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	int ret;
 
 	/* Register new destination array resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
 					    &res_idx);
-	if (!cache_resource) {
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
 				 mlx5_malloc(MLX5_MEM_ZERO,
 				 sizeof(struct mlx5dv_dr_action_dest_attr),
@@ -10823,7 +10823,7 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 			goto error;
 		}
 		dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
-		sample_act = &resource->sample_act[idx];
+		sample_act = &ctx_resource->sample_act[idx];
 		action_flags = sample_act->action_flags;
 		switch (action_flags) {
 		case MLX5_FLOW_ACTION_QUEUE:
@@ -10854,9 +10854,9 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	/* create a dest array actioin */
 	ret = mlx5_os_flow_dr_create_flow_action_dest_array
 						(domain,
-						 cache_resource->num_of_dest,
+						 resource->num_of_dest,
 						 dest_attr,
-						 &cache_resource->action);
+						 &resource->action);
 	if (ret) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10864,19 +10864,18 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 				   "cannot create destination array action");
 		goto error;
 	}
-	cache_resource->idx = res_idx;
-	cache_resource->dev = dev;
-	for (idx = 0; idx < resource->num_of_dest; idx++)
+	resource->idx = res_idx;
+	resource->dev = dev;
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 		mlx5_free(dest_attr[idx]);
-	return &cache_resource->entry;
+	return &resource->entry;
 error:
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[idx]);
+						   &resource->sample_idx[idx]);
 		if (dest_attr[idx])
 			mlx5_free(dest_attr[idx]);
 	}
-
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
 	return NULL;
 }
@@ -10886,8 +10885,8 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to destination array resource.
+ * @param[in] ref
+ *   Pointer to destination array resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10898,25 +10897,25 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_dest_array_resource *resource,
+			 struct mlx5_flow_dv_dest_array_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_dest_array = cache_resource->idx;
-	dev_flow->dv.dest_array_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_dest_array = resource->idx;
+	dev_flow->dv.dest_array_res = resource;
 	return 0;
 }
 
@@ -13345,14 +13344,15 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			  struct mlx5_cache_entry *entry)
+flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+			  struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache),
-							  entry);
+	struct mlx5_flow_dv_matcher *resource = container_of(entry,
+							     typeof(*resource),
+							     entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object));
-	mlx5_free(cache);
+	claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
+	mlx5_free(resource);
 }
 
 /**
@@ -13376,7 +13376,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13395,7 +13395,7 @@ flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
-		container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
@@ -13417,15 +13417,14 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
 				     uint32_t encap_decap_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-					encap_decap_idx);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+				  encap_decap_idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->action);
-	return mlx5_hlist_unregister(priv->sh->encaps_decaps,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
 }
 
 /**
@@ -13487,15 +13486,15 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry)
+flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_port_id_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+				  container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
 
 /**
@@ -13514,14 +13513,14 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 					uint32_t port_id)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->port_id_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13544,15 +13543,15 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry)
+flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
 
 /**
@@ -13571,15 +13570,15 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 					  struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx = handle->dvh.rix_push_vlan;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->push_vlan_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13616,26 +13615,24 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry)
+flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	if (cache_resource->verbs_action)
+	if (resource->verbs_action)
 		claim_zero(mlx5_flow_os_destroy_flow_action
-				(cache_resource->verbs_action));
-	if (cache_resource->normal_path_tbl)
+						      (resource->verbs_action));
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-			cache_resource->normal_path_tbl);
-	flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			cache_resource->idx);
-	DRV_LOG(DEBUG, "sample resource %p: removed",
-		(void *)cache_resource);
+					     resource->normal_path_tbl);
+	flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
+	DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
 }
 
 /**
@@ -13654,38 +13651,36 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 				     struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			 handle->dvh.rix_sample);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+				  handle->dvh.rix_sample);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->verbs_action);
-	return mlx5_cache_unregister(&priv->sh->sample_action_list,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->verbs_action);
+	return mlx5_list_unregister(&priv->sh->sample_action_list,
+				    &resource->entry);
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	uint32_t i = 0;
 
-	MLX5_ASSERT(cache_resource->action);
-	if (cache_resource->action)
-		claim_zero(mlx5_flow_os_destroy_flow_action
-					(cache_resource->action));
-	for (; i < cache_resource->num_of_dest; i++)
+	MLX5_ASSERT(resource->action);
+	if (resource->action)
+		claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	for (; i < resource->num_of_dest; i++)
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[i]);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			cache_resource->idx);
+						   &resource->sample_idx[i]);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
 	DRV_LOG(DEBUG, "destination array resource %p: removed",
-		(void *)cache_resource);
+		(void *)resource);
 }
 
 /**
@@ -13704,15 +13699,15 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 				    struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_dest_array_resource *cache;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			       handle->dvh.rix_dest_array);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				  handle->dvh.rix_dest_array);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->dest_array_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->dest_array_list,
+				    &resource->entry);
 }
 
 static void
@@ -14555,7 +14550,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 		if (sub_policy->color_matcher[i]) {
 			tbl = container_of(sub_policy->color_matcher[i]->tbl,
 				typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 				      &sub_policy->color_matcher[i]->entry);
 			sub_policy->color_matcher[i] = NULL;
 		}
@@ -15289,8 +15284,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_cache_unregister(&tbl->matchers,
-				      &mtrmng->def_matcher[i]->entry);
+			mlx5_list_unregister(&tbl->matchers,
+					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
 		for (j = 0; j < MLX5_REG_BITS; j++) {
@@ -15299,8 +15294,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_cache_unregister(&tbl->matchers,
-					&mtrmng->drop_matcher[i][j]->entry);
+				mlx5_list_unregister(&tbl->matchers,
+					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
 		}
@@ -15396,7 +15391,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 			bool is_default_policy,
 			struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
@@ -15432,7 +15427,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -15795,7 +15790,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 						     0, &error);
 	uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
 	uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
 			.size = sizeof(matcher.mask.buf) -
@@ -15841,7 +15836,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -15878,7 +15873,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
@@ -16064,7 +16059,6 @@ flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
 	return NULL;
 }
 
-
 /**
  * Destroy the sub policy table with RX queue.
  *
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 1b264e5994..3dcc71d51d 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index bb9a908087..8395332507 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 0ed279e162..a2b5accb84 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,29 +9,29 @@
 #include "mlx5_utils.h"
 
 
-/********************* Cache list ************************/
+/********************* MLX5 list ************************/
 
-static struct mlx5_cache_entry *
-mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry __rte_unused,
+static struct mlx5_list_entry *
+mlx5_list_default_create_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry __rte_unused,
 			     void *ctx __rte_unused)
 {
 	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
 }
 
 static void
-mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
 }
 
 int
-mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
+mlx5_list_create(struct mlx5_list *list, const char *name,
 		     uint32_t entry_size, void *ctx,
-		     mlx5_cache_create_cb cb_create,
-		     mlx5_cache_match_cb cb_match,
-		     mlx5_cache_remove_cb cb_remove)
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove)
 {
 	MLX5_ASSERT(list);
 	if (!cb_match || (!cb_create ^ !cb_remove))
@@ -40,19 +40,19 @@ mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
+	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
+	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
 	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	LIST_INIT(&list->head);
 	return 0;
 }
 
-static struct mlx5_cache_entry *
-__cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	LIST_FOREACH(entry, &list->head, next) {
 		if (list->cb_match(list, entry, ctx))
@@ -60,7 +60,7 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 		if (reuse) {
 			__atomic_add_fetch(&entry->ref_cnt, 1,
 					   __ATOMIC_RELAXED);
-			DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
 		break;
@@ -68,33 +68,33 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_cache_entry *
-cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __cache_lookup(list, ctx, reuse);
+	entry = __list_lookup(list, ctx, reuse);
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	return cache_lookup(list, ctx, false);
+	return list_lookup(list, ctx, false);
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	uint32_t prev_gen_cnt = 0;
 
 	MLX5_ASSERT(list);
 	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
 	/* Lookup with read lock, reuse if found. */
-	entry = cache_lookup(list, ctx, true);
+	entry = list_lookup(list, ctx, true);
 	if (entry)
 		return entry;
 	/* Not found, append with write lock - block read from other threads. */
@@ -102,13 +102,13 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __cache_lookup(list, ctx, true);
+		entry = __list_lookup(list, ctx, true);
 		if (entry)
 			goto done;
 	}
 	entry = list->cb_create(list, entry, ctx);
 	if (!entry) {
-		DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
+		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
 			list->name, (void *)entry);
 		goto done;
 	}
@@ -116,7 +116,7 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	LIST_INSERT_HEAD(&list->head, entry, next);
 	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 done:
 	rte_rwlock_write_unlock(&list->lock);
@@ -124,12 +124,12 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 }
 
 int
-mlx5_cache_unregister(struct mlx5_cache_list *list,
-		      struct mlx5_cache_entry *entry)
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
 {
 	rte_rwlock_write_lock(&list->lock);
 	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 	if (--entry->ref_cnt) {
 		rte_rwlock_write_unlock(&list->lock);
@@ -140,15 +140,15 @@ mlx5_cache_unregister(struct mlx5_cache_list *list,
 	LIST_REMOVE(entry, next);
 	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 		list->name, (void *)entry);
 	return 0;
 }
 
 void
-mlx5_cache_list_destroy(struct mlx5_cache_list *list)
+mlx5_list_destroy(struct mlx5_list *list)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	MLX5_ASSERT(list);
 	/* no LIST_FOREACH_SAFE, using while instead */
@@ -156,14 +156,14 @@ mlx5_cache_list_destroy(struct mlx5_cache_list *list)
 		entry = LIST_FIRST(&list->head);
 		LIST_REMOVE(entry, next);
 		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
 			list->name, (void *)entry);
 	}
 	memset(list, 0, sizeof(*list));
 }
 
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
+mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
 	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 737dd7052d..593793345d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -296,19 +296,19 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ cache list *****************************/
+/************************ mlx5 list *****************************/
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
 
-struct mlx5_cache_list;
+struct mlx5_list;
 
 /**
- * Structure of the entry in the cache list, user should define its own struct
+ * Structure of the entry in the mlx5 list, user should define its own struct
  * that contains this in order to store the data.
  */
-struct mlx5_cache_entry {
-	LIST_ENTRY(mlx5_cache_entry) next; /* Entry pointers in the list. */
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* Reference count. */
 };
 
@@ -316,18 +316,18 @@ struct mlx5_cache_entry {
  * Type of callback function for entry removal.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
-				     struct mlx5_cache_entry *entry);
+typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
+				     struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -336,14 +336,14 @@ typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
-				   struct mlx5_cache_entry *entry, void *ctx);
+typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+				   struct mlx5_list_entry *entry, void *ctx);
 
 /**
- * Type of function for user defined cache list entry creation.
+ * Type of function for user defined mlx5 list entry creation.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The new allocated entry, NULL if list entry size unspecified,
  *   New entry has to be allocated in callback and return.
@@ -353,46 +353,46 @@ typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_cache_entry *(*mlx5_cache_create_cb)
-				 (struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry,
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry,
 				  void *ctx);
 
 /**
- * Linked cache list structure.
+ * Linked mlx5 list structure.
  *
- * Entry in cache list could be reused if entry already exists,
+ * Entry in mlx5 list could be reused if entry already exists,
  * reference count will increase and the existing entry returns.
  *
  * When destroy an entry from list, decrease reference count and only
  * destroy when no further reference.
  *
- * Linked list cache is designed for limited number of entries cache,
+ * Linked list is designed for limited number of entries,
  * read mostly, less modification.
  *
- * For huge amount of entries cache, please consider hash list cache.
+ * For huge amount of entries, please consider hash list.
  *
  */
-struct mlx5_cache_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the cache list. */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	mlx5_cache_create_cb cb_create; /**< entry create callback. */
-	mlx5_cache_match_cb cb_match; /**< entry match callback. */
-	mlx5_cache_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_cache_head, mlx5_cache_entry) head;
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
 };
 
 /**
- * Initialize a cache list.
+ * Create a mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
  * @param name
- *   Name of the cache list.
+ *   Name of the mlx5 list.
  * @param entry_size
  *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
@@ -406,11 +406,11 @@ struct mlx5_cache_list {
  * @return
  *   0 on success, otherwise failure.
  */
-int mlx5_cache_list_init(struct mlx5_cache_list *list,
+int mlx5_list_create(struct mlx5_list *list,
 			 const char *name, uint32_t entry_size, void *ctx,
-			 mlx5_cache_create_cb cb_create,
-			 mlx5_cache_match_cb cb_match,
-			 mlx5_cache_remove_cb cb_remove);
+			 mlx5_list_create_cb cb_create,
+			 mlx5_list_match_cb cb_match,
+			 mlx5_list_remove_cb cb_remove);
 
 /**
  * Search an entry matching the key.
@@ -419,18 +419,18 @@ int mlx5_cache_list_init(struct mlx5_cache_list *list,
  * this function only in main thread.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  * @param ctx
  *   Common context parameter used by entry callback function.
  *
  * @return
- *   Pointer of the cache entry if found, NULL otherwise.
+ *   Pointer of the list entry if found, NULL otherwise.
  */
-struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
 					   void *ctx);
 
 /**
- * Reuse or create an entry to the cache list.
+ * Reuse or create an entry to the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
@@ -440,42 +440,42 @@ struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
  * @return
  *   registered entry on success, NULL otherwise
  */
-struct mlx5_cache_entry *mlx5_cache_register(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
 					     void *ctx);
 
 /**
- * Remove an entry from the cache list.
+ * Remove an entry from the mlx5 list.
  *
  * User should guarantee the validity of the entry.
  *
  * @param list
  *   Pointer to the hast list.
  * @param entry
- *   Entry to be removed from the cache list table.
+ *   Entry to be removed from the mlx5 list table.
  * @return
  *   0 on entry removed, 1 on entry still referenced.
  */
-int mlx5_cache_unregister(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry);
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
 
 /**
- * Destroy the cache list.
+ * Destroy the mlx5 list.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  */
-void mlx5_cache_list_destroy(struct mlx5_cache_list *list);
+void mlx5_list_destroy(struct mlx5_list *list);
 
 /**
- * Get entry number from the cache list.
+ * Get entry number from the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list.
  * @return
- *   Cache list entry number.
+ *   mlx5 list entry number.
  */
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list);
+mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /********************************* indexed pool *************************/
 
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 7d15c998bb..b10c47fee3 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -608,10 +608,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v3 07/22] net/mlx5: add per lcore cache to the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When mlx5 list object is accessed by multiple cores, the list lock
counter is all the time written by all the cores what increases cache
misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup
operation, all the other threads coming to do an operation in the list
are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when
the list objects are mostly reused.

Synchronization with atomic operations should be done in order to
allow threads to unregister an entry from other thread cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 9aa57e38b7..8a043526da 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1702,10 +1710,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 58646da331..546bee761e 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1365,6 +1365,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 4dec703366..ce363355c1 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1634,7 +1634,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1642,6 +1646,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1650,6 +1659,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1658,6 +1672,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 897bcf52a6..68a9e70a98 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3820,6 +3820,39 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_port_id_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate port_id action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
+}
+
 /**
  * Find existing table port ID resource or create and register a new one.
  *
@@ -3912,6 +3945,39 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate push_vlan action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
+}
+
 /**
  * Find existing push vlan resource or create and register a new one.
  *
@@ -9848,6 +9914,36 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 	return match_criteria_enable;
 }
 
+static struct mlx5_list_entry *
+flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_matcher *ref = ctx->data;
+	struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
+							    typeof(*tbl), tbl);
+	struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
+							    sizeof(*resource),
+							    0, SOCKET_ID_ANY);
+
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot create matcher");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->tbl = &tbl->tbl;
+	return &resource->entry;
+}
+
+static void
+flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 struct mlx5_hlist_entry *
 flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 {
@@ -9914,10 +10010,12 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
 			 flow_dv_matcher_create_cb,
 			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb);
+			 flow_dv_matcher_remove_cb,
+			 flow_dv_matcher_clone_cb,
+			 flow_dv_matcher_clone_free_cb);
 	return &tbl_data->entry;
 }
 
@@ -10705,6 +10803,45 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 
 }
 
+struct mlx5_list_entry *
+flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t idx = 0;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_sample_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+			resource->idx);
+}
+
 /**
  * Find existing sample resource or create and register a new one.
  *
@@ -10880,6 +11017,46 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 	return NULL;
 }
 
+struct mlx5_list_entry *
+flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t res_idx = 0;
+	struct rte_flow_error *error = ctx->error;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				      &res_idx);
+	if (!resource) {
+		rte_flow_error_set(error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate dest-array memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = res_idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
+}
+
 /**
  * Find existing destination array resource or create and register a new one.
  *
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 3dcc71d51d..5450ddd388 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 8395332507..f8769da8dc 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1857,20 +1857,18 @@ mlx5_ind_table_obj_get(struct rte_eth_dev *dev, const uint16_t *queues,
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_ind_table_obj *ind_tbl;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		if ((ind_tbl->queues_n == queues_n) &&
 		    (memcmp(ind_tbl->queues, queues,
 			    ind_tbl->queues_n * sizeof(ind_tbl->queues[0]))
-		     == 0))
+		     == 0)) {
+			__atomic_fetch_add(&ind_tbl->refcnt, 1,
+					   __ATOMIC_RELAXED);
 			break;
+		}
 	}
-	if (ind_tbl) {
-		unsigned int i;
-
-		__atomic_fetch_add(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
-		for (i = 0; i != ind_tbl->queues_n; ++i)
-			mlx5_rxq_get(dev, ind_tbl->queues[i]);
-	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ind_tbl;
 }
 
@@ -1893,19 +1891,20 @@ mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
 			   bool standalone)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	unsigned int i;
+	unsigned int i, ret;
 
-	if (__atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED) == 0)
-		priv->obj_ops.ind_table_destroy(ind_tbl);
+	rte_rwlock_write_lock(&priv->ind_tbls_lock);
+	ret = __atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
+	if (!ret && !standalone)
+		LIST_REMOVE(ind_tbl, next);
+	rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	if (ret)
+		return 1;
+	priv->obj_ops.ind_table_destroy(ind_tbl);
 	for (i = 0; i != ind_tbl->queues_n; ++i)
 		claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
-	if (__atomic_load_n(&ind_tbl->refcnt, __ATOMIC_RELAXED) == 0) {
-		if (!standalone)
-			LIST_REMOVE(ind_tbl, next);
-		mlx5_free(ind_tbl);
-		return 0;
-	}
-	return 1;
+	mlx5_free(ind_tbl);
+	return 0;
 }
 
 /**
@@ -1924,12 +1923,14 @@ mlx5_ind_table_obj_verify(struct rte_eth_dev *dev)
 	struct mlx5_ind_table_obj *ind_tbl;
 	int ret = 0;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		DRV_LOG(DEBUG,
 			"port %u indirection table obj %p still referenced",
 			dev->data->port_id, (void *)ind_tbl);
 		++ret;
 	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ret;
 }
 
@@ -2015,8 +2016,11 @@ mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues,
 		mlx5_free(ind_tbl);
 		return NULL;
 	}
-	if (!standalone)
+	if (!standalone) {
+		rte_rwlock_write_lock(&priv->ind_tbls_lock);
 		LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
+		rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	}
 	return ind_tbl;
 }
 
@@ -2328,6 +2332,35 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 	return hrxq ? &hrxq->entry : NULL;
 }
 
+struct mlx5_list_entry *
+mlx5_hrxq_clone_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry,
+		    void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq;
+	uint32_t hrxq_idx = 0;
+
+	hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx);
+	if (!hrxq)
+		return NULL;
+	memcpy(hrxq, entry, sizeof(*hrxq) + MLX5_RSS_HASH_KEY_LEN);
+	hrxq->idx = hrxq_idx;
+	return &hrxq->entry;
+}
+
+void
+mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq->idx);
+}
+
 /**
  * Get an Rx Hash queue.
  *
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index a2b5accb84..51cca68ea9 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,57 +9,68 @@
 #include "mlx5_utils.h"
 
 
-/********************* MLX5 list ************************/
-
-static struct mlx5_list_entry *
-mlx5_list_default_create_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
-{
-	mlx5_free(entry);
-}
+/********************* mlx5 list ************************/
 
 int
-mlx5_list_create(struct mlx5_list *list, const char *name,
-		     uint32_t entry_size, void *ctx,
-		     mlx5_list_create_cb cb_create,
-		     mlx5_list_match_cb cb_match,
-		     mlx5_list_remove_cb cb_remove)
+mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	int i;
+
 	MLX5_ASSERT(list);
-	if (!cb_match || (!cb_create ^ !cb_remove))
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free)
 		return -1;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
+	list->cb_create = cb_create;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	LIST_INIT(&list->head);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry;
-
-	LIST_FOREACH(entry, &list->head, next) {
-		if (list->cb_match(list, entry, ctx))
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (list->cb_match(list, entry, ctx)) {
+			if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_ACQUIRE);
+				if (ret == 0) {
+					LIST_REMOVE(entry, next);
+					list->cb_clone_free(list, entry);
+				}
+			}
+			entry = nentry;
 			continue;
+		}
 		if (reuse) {
-			__atomic_add_fetch(&entry->ref_cnt, 1,
-					   __ATOMIC_RELAXED);
+			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+						 __ATOMIC_ACQUIRE);
+			if (ret == 1u) {
+				/* Entry was invalid before, free it. */
+				LIST_REMOVE(entry, next);
+				list->cb_clone_free(list, entry);
+				entry = nentry;
+				continue;
+			}
 			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
@@ -68,96 +79,141 @@ __list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_list_entry *
-list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry = NULL;
+	int i;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, ctx, reuse);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	return list_lookup(list, ctx, false);
+	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+
+	if (!lentry)
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
 }
 
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry, *lentry;
 	uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
-	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* Lookup in local cache. */
+	lentry = __list_lookup(list, lcore_index, ctx, true);
+	if (lentry)
+		return lentry;
 	/* Lookup with read lock, reuse if found. */
-	entry = list_lookup(list, ctx, true);
-	if (entry)
-		return entry;
+	rte_rwlock_read_lock(&list->lock);
+	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+	if (entry == NULL) {
+		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
+					       __ATOMIC_ACQUIRE);
+		rte_rwlock_read_unlock(&list->lock);
+	} else {
+		rte_rwlock_read_unlock(&list->lock);
+		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+	}
 	/* Not found, append with write lock - block read from other threads. */
 	rte_rwlock_write_lock(&list->lock);
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, ctx, true);
-		if (entry)
-			goto done;
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (entry) {
+			rte_rwlock_write_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
 	}
 	entry = list->cb_create(list, entry, ctx);
-	if (!entry) {
-		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
-			list->name, (void *)entry);
-		goto done;
+	if (entry) {
+		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+		if (!lentry) {
+			list->cb_remove(list, entry);
+		} else {
+			entry->ref_cnt = 1u;
+			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
+					 next);
+			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
+			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+				list->name, (void *)entry, entry->ref_cnt);
+		}
+
 	}
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(&list->head, entry, next);
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-done:
 	rte_rwlock_write_unlock(&list->lock);
-	return entry;
+	return lentry;
 }
 
 int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
+	struct mlx5_list_entry *gentry = entry->gentry;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
+	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
+		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		LIST_REMOVE(gentry, next);
+		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
-		return 1;
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
 	}
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-	__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	LIST_REMOVE(entry, next);
-	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-		list->name, (void *)entry);
-	return 0;
+	return 1;
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
+	int i;
 
 	MLX5_ASSERT(list);
-	/* no LIST_FOREACH_SAFE, using while instead */
-	while (!LIST_EMPTY(&list->head)) {
-		entry = LIST_FIRST(&list->head);
-		LIST_REMOVE(entry, next);
-		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
-			list->name, (void *)entry);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list, entry);
+			}
+		}
 	}
 	memset(list, 0, sizeof(*list));
 }
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 593793345d..24ae2b2ccb 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -309,9 +309,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -339,6 +344,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -375,15 +387,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -393,8 +407,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -407,10 +419,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index b10c47fee3..f6cf1928b2 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -608,9 +608,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v3 08/22] net/mlx5: minimize list critical sections

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is
taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical
sections.

These operations are heavy and make the critical sections heavy because
they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation
counter in order to detect parallel allocations.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 51cca68ea9..772b352af5 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 24ae2b2ccb..684d1e8a2a 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -387,8 +387,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
-- 
2.25.1

[dpdk-dev] [PATCH v3 09/22] net/mlx5: manage list cache entries release

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When a cache entry is allocated by lcore A and is released by lcore B,
the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be
increased atomically when the non-original lcore decreases the reference
counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in
its cache, it will check the counter in order to free invalid entries in
its cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 772b352af5..7cdf44dcf7 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 684d1e8a2a..ffa9cd5142 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -310,11 +310,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v3 10/22] net/mlx5: relax the list utility atomic operations

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The atomic operation in the list utility no need a barriers because the
critical part are managed by RW lock.

Relax them.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 7cdf44dcf7..29248c80ed 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
-- 
2.25.1

[dpdk-dev] [PATCH v3 11/22] net/mlx5: allocate list memory by the create API

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence
with the hlist utility.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  56 ++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 137 insertions(+), 81 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 8a043526da..87b63d852b 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -447,6 +455,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -508,9 +532,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1710,11 +1748,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1771,7 +1811,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 9aade013c5..775ea15adb 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1611,7 +1611,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 546bee761e..df5cba3d45 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ce363355c1..ce4d205e86 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 68a9e70a98..d588e6fd37 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3883,7 +3883,7 @@ flow_dv_port_id_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -4008,7 +4008,7 @@ flow_dv_push_vlan_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10010,12 +10010,22 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
-			 flow_dv_matcher_create_cb,
-			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb,
-			 flow_dv_matcher_clone_cb,
-			 flow_dv_matcher_clone_free_cb);
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+					      flow_dv_matcher_create_cb,
+					      flow_dv_matcher_match_cb,
+					      flow_dv_matcher_remove_cb,
+					      flow_dv_matcher_clone_cb,
+					      flow_dv_matcher_clone_free_cb);
+	if (!tbl_data->matchers) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot create tbl matcher list");
+		mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
+		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
+		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
+		return NULL;
+	}
 	return &tbl_data->entry;
 }
 
@@ -10143,7 +10153,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10275,7 +10285,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
@@ -10872,7 +10882,7 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -11087,7 +11097,7 @@ flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -13553,7 +13563,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13696,7 +13706,7 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+	return mlx5_list_unregister(priv->sh->port_id_action_list,
 				    &resource->entry);
 }
 
@@ -13754,7 +13764,7 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+	return mlx5_list_unregister(priv->sh->push_vlan_action_list,
 				    &resource->entry);
 }
 
@@ -13835,7 +13845,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->verbs_action);
-	return mlx5_list_unregister(&priv->sh->sample_action_list,
+	return mlx5_list_unregister(priv->sh->sample_action_list,
 				    &resource->entry);
 }
 
@@ -13883,7 +13893,7 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->dest_array_list,
+	return mlx5_list_unregister(priv->sh->dest_array_list,
 				    &resource->entry);
 }
 
@@ -14727,7 +14737,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 		if (sub_policy->color_matcher[i]) {
 			tbl = container_of(sub_policy->color_matcher[i]->tbl,
 				typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 				      &sub_policy->color_matcher[i]->entry);
 			sub_policy->color_matcher[i] = NULL;
 		}
@@ -15461,7 +15471,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
@@ -15471,7 +15481,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_list_unregister(&tbl->matchers,
+				mlx5_list_unregister(tbl->matchers,
 					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
@@ -15604,7 +15614,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -16013,7 +16023,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -16050,7 +16060,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index f8769da8dc..aa9e973d10 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 29248c80ed..a4526444f9 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index ffa9cd5142..0bf2f5f5ca 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -419,15 +419,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index f6cf1928b2..97a8f04e39 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -608,7 +608,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v3 12/22] common/mlx5: add per-lcore cache to hash list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates
all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and
hlist.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 doc/guides/nics/mlx5.rst                |   5 +
 doc/guides/rel_notes/release_21_08.rst  |   6 +
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 466 +++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h | 267 ++++++++++----
 drivers/common/mlx5/version.map         |   7 +
 drivers/net/mlx5/linux/mlx5_os.c        |  46 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++---
 drivers/net/mlx5/mlx5_flow.h            | 185 +++++-----
 drivers/net/mlx5/mlx5_flow_dv.c         | 407 ++++++++++++---------
 drivers/net/mlx5/mlx5_rx.h              |  13 +-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +--
 drivers/net/mlx5/mlx5_utils.c           | 251 -------------
 drivers/net/mlx5/mlx5_utils.h           | 197 ----------
 drivers/net/mlx5/windows/mlx5_os.c      |   2 +-
 17 files changed, 1029 insertions(+), 1044 deletions(-)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index eb44a070b1..9bd3846e0d 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -445,6 +445,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..f6cd1d137d 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Updated Mellanox mlx5 driver.**
+
+  Updated the Mellanox mlx5 driver with new features and improvements, including:
+
+  * Optimize multiple-thread flow insertion rate.
+
 
 Removed Items
 -------------
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..1809ff1e95 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index ad2011e858..4e385c616a 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -11,39 +11,324 @@
 #include "mlx5_common_utils.h"
 #include "mlx5_common_log.h"
 
-/********************* Hash List **********************/
+/********************* mlx5 list ************************/
+
+static int
+mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
+	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       mlx5_list_match_cb cb_match,
+	       mlx5_list_remove_cb cb_remove,
+	       mlx5_list_clone_cb cb_clone,
+	       mlx5_list_clone_free_cb cb_clone_free)
+{
+	int i;
+
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return -EINVAL;
+	}
+	if (name)
+		snprintf(list->name, sizeof(list->name), "%s", name);
+	list->ctx = ctx;
+	list->lcores_share = lcores_share;
+	list->cb_create = cb_create;
+	list->cb_match = cb_match;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
+	rte_rwlock_init(&list->lock);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
+	return 0;
+}
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (mlx5_list_init(list, name, ctx, lcores_share,
+			   cb_create, cb_match, cb_remove, cb_clone,
+			   cb_clone_free) != 0) {
+		mlx5_free(list);
+		return NULL;
+	}
+	return list;
+}
+
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+{
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_RELAXED) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_RELAXED);
+			}
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
+		}
+		entry = LIST_NEXT(entry, next);
+	}
+	return NULL;
+}
+
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry = NULL;
+	int i;
+
+	rte_rwlock_read_lock(&list->lock);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
+	rte_rwlock_read_unlock(&list->lock);
+	return entry;
+}
+
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
+{
+	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+
+	if (unlikely(!lentry))
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
+}
+
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			if (list->lcores_share)
+				list->cb_clone_free(list->ctx, entry);
+			else
+				list->cb_remove(list->ctx, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	MLX5_ASSERT(list);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
+		rte_rwlock_read_unlock(&list->lock);
+	}
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list->ctx, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+		return entry;
+	}
+	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list->ctx, entry);
+		return NULL;
+	}
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
+	rte_rwlock_write_lock(&list->lock);
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
+			rte_rwlock_write_unlock(&list->lock);
+			list->cb_remove(list->ctx, entry);
+			list->cb_clone_free(list->ctx, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
+		}
+	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
+	rte_rwlock_write_unlock(&list->lock);
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
+	return local_entry;
+}
 
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
 {
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
+	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		if (list->lcores_share)
+			list->cb_clone_free(list->ctx, entry);
+		else
+			list->cb_remove(list->ctx, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	rte_rwlock_write_lock(&list->lock);
+	if (likely(gentry->ref_cnt == 0)) {
+		LIST_REMOVE(gentry, next);
+		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list->ctx, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
+	}
+	rte_rwlock_write_unlock(&list->lock);
+	return 1;
 }
 
 static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
+mlx5_list_uninit(struct mlx5_list *list)
+{
+	struct mlx5_list_entry *entry;
+	int i;
+
+	MLX5_ASSERT(list);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list->ctx, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list->ctx, entry);
+			}
+		}
+	}
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	mlx5_list_uninit(list);
+	mlx5_free(list);
+}
+
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list)
 {
-	mlx5_free(entry);
+	MLX5_ASSERT(list);
+	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
 }
 
+/********************* Hash List **********************/
+
 struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
+		  bool lcores_share, void *ctx, mlx5_list_create_cb cb_create,
+		  mlx5_list_match_cb cb_match,
+		  mlx5_list_remove_cb cb_remove,
+		  mlx5_list_clone_cb cb_clone,
+		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+		DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
+			"be aligned to 0x%" PRIX32 ".", size, act_size);
 	} else {
 		act_size = size;
 	}
@@ -57,61 +342,24 @@ mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 			name ? name : "None");
 		return NULL;
 	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
 	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
+	h->lcores_share = lcores_share;
+	h->direct_key = direct_key;
+	for (i = 0; i < act_size; i++) {
+		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   cb_create, cb_match, cb_remove, cb_clone,
+				   cb_clone_free) != 0) {
+			mlx5_free(h);
+			return NULL;
 		}
 	}
-	return node;
+	DRV_LOG(DEBUG, "Hash list %s with size 0x%" PRIX32 " was created.",
+		name, act_size);
+	return h;
 }
 
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
 
-struct mlx5_hlist_entry *
+struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
@@ -120,102 +368,44 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
+	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
 }
 
-struct mlx5_hlist_entry*
+struct mlx5_list_entry*
 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry;
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
+	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	if (likely(entry)) {
+		if (h->lcores_share)
+			entry->gentry->bucket_idx = idx;
+		else
+			entry->bucket_idx = idx;
 	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
-	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
 	return entry;
 }
 
 int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = entry->idx;
+	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+							      entry->bucket_idx;
 
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
+	return mlx5_list_unregister(&h->buckets[idx].l, entry);
 }
 
 void
 mlx5_hlist_destroy(struct mlx5_hlist *h)
 {
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
+	uint32_t i;
 
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
+	for (i = 0; i <= h->mask; i++)
+		mlx5_list_uninit(&h->buckets[i].l);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index ed378ce9bd..61b30a45ca 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -7,106 +7,227 @@
 
 #include "mlx5_common.h"
 
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
+/************************ mlx5 list *****************************/
 
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE			32
+/** Maximum size of string for naming. */
+#define MLX5_NAME_SIZE			32
 
-struct mlx5_hlist;
+struct mlx5_list;
 
 /**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
+ * Structure of the entry in the mlx5 list, user should define its own struct
+ * that contains this in order to store the data.
  */
-struct mlx5_hlist_entry {
-	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint32_t idx; /* Bucket index the entry belongs to. */
-	uint32_t ref_cnt; /* Reference count. */
-};
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
+	uint32_t ref_cnt __rte_aligned(8); /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
+	union {
+		struct mlx5_list_entry *gentry;
+		uint32_t bucket_idx;
+	};
+} __rte_packed;
 
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
+} __rte_cache_aligned;
 
 /**
  * Type of callback function for entry removal.
  *
- * @param list
- *   The hash list.
+ * @param tool_ctx
+ *   The tool instance user context.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
-				     struct mlx5_hlist_entry *entry);
+typedef void (*mlx5_list_remove_cb)(void *tool_ctx,
+				    struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
- * @param list
- *   The hash list.
+ * @param tool_ctx
+ *   The tool instance context.
  * @param entry
  *   The entry in the list.
- * @param key
- *   The new entry key.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry,
-				   uint64_t key, void *ctx);
+typedef int (*mlx5_list_match_cb)(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *ctx);
+
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)(void *tool_ctx,
+				      struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
+					struct mlx5_list_entry *entry);
 
 /**
- * Type of function for user defined hash list entry creation.
+ * Type of function for user defined mlx5 list entry creation.
  *
- * @param list
- *   The hash list.
- * @param key
- *   The key of the new entry.
+ * @param tool_ctx
+ *   The mlx5 tool instance context.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
- *   Pointer to allocated entry on success, NULL otherwise.
+ *   Pointer of entry on success, NULL otherwise.
+ */
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
+						       void *ctx);
+
+/**
+ * Linked mlx5 list structure.
+ *
+ * Entry in mlx5 list could be reused if entry already exists,
+ * reference count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ *
+ * Linked list is designed for limited number of entries,
+ * read mostly, less modification.
+ *
+ * For huge amount of entries, please consider hash list.
+ *
+ */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	void *ctx; /* user objects target to callback. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
+};
+
+/**
+ * Create a mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param name
+ *   Name of the mlx5 list.
+ * @param ctx
+ *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
+ * @param cb_create
+ *   Callback function for entry create.
+ * @param cb_match
+ *   Callback function for entry match.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @return
+ *   List pointer on success, otherwise NULL.
+ */
+__rte_internal
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ * @param ctx
+ *   Common context parameter used by entry callback function.
+ *
+ * @return
+ *   Pointer of the list entry if found, NULL otherwise.
  */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
-				  (struct mlx5_hlist *list,
-				   uint64_t key, void *ctx);
+__rte_internal
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
+					   void *ctx);
 
-/* Hash list bucket head. */
+/**
+ * Reuse or create an entry to the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param ctx
+ *   Common context parameter used by callback function.
+ *
+ * @return
+ *   registered entry on success, NULL otherwise
+ */
+__rte_internal
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
+					     void *ctx);
+
+/**
+ * Remove an entry from the mlx5 list.
+ *
+ * User should guarantee the validity of the entry.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @param entry
+ *   Entry to be removed from the mlx5 list table.
+ * @return
+ *   0 on entry removed, 1 on entry still referenced.
+ */
+__rte_internal
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
+
+/**
+ * Destroy the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ */
+__rte_internal
+void mlx5_list_destroy(struct mlx5_list *list);
+
+/**
+ * Get entry number from the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @return
+ *   mlx5 list entry number.
+ */
+__rte_internal
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list);
+
+/********************* Hash List **********************/
+
+/* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_hlist_head head; /* List head. */
-	rte_rwlock_t lock; /* Bucket lock. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
+	struct mlx5_list l;
 } __rte_cache_aligned;
 
 /**
  * Hash list table structure
  *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
+ * The hash list bucket using the mlx5_list object for managing.
  */
 struct mlx5_hlist {
-	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
-	/**< number of heads, need to be power of 2. */
-	uint32_t table_sz;
-	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
-	/**< mask to get the index of the list heads. */
-	uint32_t mask;
-	bool direct_key; /* Use the new entry key directly as hash index. */
-	bool write_most; /* List mostly used for append new or destroy. */
-	void *ctx;
-	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
-	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
-	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+	uint32_t mask; /* A mask for the bucket index range. */
+	uint8_t flags;
+	bool direct_key; /* Whether to use the key directly as hash index. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
-	/**< list bucket arrays. */
 };
 
 /**
@@ -123,23 +244,33 @@ struct mlx5_hlist {
  *   Heads array size of the hash list.
  * @param entry_size
  *   Entry size to allocate if cb_create not specified.
- * @param flags
- *   The hash list attribute flags.
+ * @param direct key
+ *   Whether to use the key directly as hash index.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
+ * @param ctx
+ *   The hlist instance context.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
  *   Callback function for entry match.
- * @param cb_destroy
- *   Callback function for entry destroy.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @param cb_clone
+ *   Callback function for entry clone.
+ * @param cb_clone_free
+ *   Callback function for entry clone free.
  * @return
  *   Pointer of the hash list table created, NULL on failure.
  */
 __rte_internal
 struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
-				     uint32_t entry_size, uint32_t flags,
-				     mlx5_hlist_create_cb cb_create,
-				     mlx5_hlist_match_cb cb_match,
-				     mlx5_hlist_remove_cb cb_destroy);
+				     bool direct_key, bool lcores_share,
+				     void *ctx, mlx5_list_create_cb cb_create,
+				     mlx5_list_match_cb cb_match,
+				     mlx5_list_remove_cb cb_remove,
+				     mlx5_list_clone_cb cb_clone,
+				     mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
@@ -158,7 +289,7 @@ struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
  *   Pointer of the hlist entry if found, NULL otherwise.
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
 					   void *ctx);
 
 /**
@@ -177,7 +308,7 @@ struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
  *   registered entry on success, NULL otherwise
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
 					     void *ctx);
 
 /**
@@ -192,7 +323,7 @@ struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
  *   0 on entry removed, 1 on entry still referenced.
  */
 __rte_internal
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry);
 
 /**
  * Destroy the hash list table, all the entries already inserted into the lists
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index b8be73a77b..e6586d6f6f 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -73,6 +73,13 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_list_create;
+	mlx5_list_register;
+	mlx5_list_unregister;
+	mlx5_list_lookup;
+	mlx5_list_get_entry_num;
+	mlx5_list_destroy;
+
 	mlx5_hlist_create;
 	mlx5_hlist_lookup;
 	mlx5_hlist_register;
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 87b63d852b..cf573a9a4d 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -261,7 +261,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+	char s[MLX5_NAME_SIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 						   flow_dv_port_id_create_cb,
 						   flow_dv_port_id_match_cb,
 						   flow_dv_port_id_remove_cb,
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 						    flow_dv_push_vlan_create_cb,
 						    flow_dv_push_vlan_match_cb,
 						    flow_dv_push_vlan_remove_cb,
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 						  flow_dv_sample_remove_cb,
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -314,44 +314,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb,
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
+					  false, sh, flow_dv_tag_create_cb,
 					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb);
+					  flow_dv_tag_remove_cb,
+					  flow_dv_tag_clone_cb,
+					  flow_dv_tag_clone_free_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, MLX5_HLIST_WRITE_MOST |
-					    MLX5_HLIST_DIRECT_KEY,
+					    true, false, sh,
 					    flow_dv_modify_create_cb,
 					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb);
+					    flow_dv_modify_remove_cb,
+					    flow_dv_modify_clone_cb,
+					    flow_dv_modify_clone_free_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, MLX5_HLIST_DIRECT_KEY |
-					      MLX5_HLIST_WRITE_MOST,
+					      true, true, sh,
 					      flow_dv_encap_decap_create_cb,
 					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb);
+					      flow_dv_encap_decap_remove_cb,
+					      flow_dv_encap_decap_clone_cb,
+					     flow_dv_encap_decap_clone_free_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -1748,7 +1748,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
 				       mlx5_hrxq_clone_cb,
@@ -1780,15 +1781,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
-						      0, 0,
+						      false, true, eth_dev,
 						      flow_dv_mreg_create_cb,
 						      flow_dv_mreg_match_cb,
-						      flow_dv_mreg_remove_cb);
+						      flow_dv_mreg_remove_cb,
+						      flow_dv_mreg_clone_cb,
+						    flow_dv_mreg_clone_free_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
-		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 775ea15adb..0e80408511 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1358,20 +1358,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index df5cba3d45..f3768ee028 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 450a84a6c5..7bd45d3895 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3983,28 +3983,27 @@ flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
-flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
-		      struct mlx5_hlist_entry *entry,
-		      uint64_t key, void *cb_ctx __rte_unused)
+flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
+		      struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
 
-	return mcp_res->mark_id != key;
+	return mcp_res->mark_id != *(uint32_t *)(ctx->data);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
-		       void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
 	struct rte_flow_error *error = ctx->error;
 	uint32_t idx = 0;
 	int ret;
-	uint32_t mark_id = key;
+	uint32_t mark_id = *(uint32_t *)(ctx->data);
 	struct rte_flow_attr attr = {
 		.group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
 		.ingress = 1,
@@ -4110,6 +4109,36 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return &mcp_res->hlist_ent;
 }
 
+struct mlx5_list_entry *
+flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	uint32_t idx = 0;
+
+	mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
+	if (!mcp_res) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	memcpy(mcp_res, oentry, sizeof(*mcp_res));
+	mcp_res->idx = idx;
+	return &mcp_res->hlist_ent;
+}
+
+void
+flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_mreg_copy_resource *mcp_res =
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
+}
+
 /**
  * Add a flow of copying flow metadata registers in RX_CP_TBL.
  *
@@ -4140,10 +4169,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 			  struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
+		.data = &mark_id,
 	};
 
 	/* Check if already registered. */
@@ -4156,11 +4186,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 }
 
 void
-flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
+flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
-	struct rte_eth_dev *dev = list->ctx;
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
@@ -4206,14 +4236,17 @@ flow_mreg_del_copy_action(struct rte_eth_dev *dev,
 static void
 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
 {
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check if default flow is registered. */
 	if (!priv->mreg_cp_tbl)
 		return;
-	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
-				  MLX5_DEFAULT_COPY_ID, NULL);
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
 	if (!entry)
 		return;
 	mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
@@ -4239,6 +4272,8 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check whether extensive metadata feature is engaged. */
 	if (!priv->config.dv_flow_en ||
@@ -4250,9 +4285,11 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 	 * Add default mreg copy flow may be called multiple time, but
 	 * only be called once in stop. Avoid register it twice.
 	 */
-	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
 		return 0;
-	mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
+	mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
 	if (!mcp_res)
 		return -rte_errno;
 	return 0;
@@ -8350,7 +8387,7 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	union tunnel_offload_mark mbits = { .val = mark };
 	union mlx5_flow_tbl_key table_key = {
 		{
@@ -8362,16 +8399,20 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 			.is_egress = 0,
 		}
 	};
-	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &table_key.v64,
+	};
+
+	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
 	return he ?
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
 
 static void
-mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry)
+mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
@@ -8380,26 +8421,26 @@ mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
 }
 
 static int
-mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-				  struct mlx5_hlist_entry *entry,
-				  uint64_t key, void *cb_ctx __rte_unused)
+mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
 }
 
-static struct mlx5_hlist_entry *
-mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
-				   void *ctx __rte_unused)
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 
 	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
@@ -8428,13 +8469,36 @@ mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return NULL;
 }
 
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *oentry,
+				  void *cb_ctx __rte_unused)
+{
+	struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
+						   0, SOCKET_ID_ANY);
+
+	if (!tte)
+		return NULL;
+	memcpy(tte, oentry, sizeof(*tte));
+	return &tte->hash;
+}
+
+static void
+mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
+				       struct mlx5_list_entry *entry)
+{
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	mlx5_free(tte);
+}
+
 static uint32_t
 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 				const struct mlx5_flow_tunnel *tunnel,
 				uint32_t group, uint32_t *table,
 				struct rte_flow_error *error)
 {
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key key = {
 		.tunnel_id = tunnel ? tunnel->tunnel_id : 0,
@@ -8442,9 +8506,12 @@ tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 	};
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_hlist *group_hash;
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &key.val,
+	};
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_register(group_hash, key.val, NULL);
+	he = mlx5_hlist_register(group_hash, key.val, &ctx);
 	if (!he)
 		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
@@ -8558,15 +8625,17 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
-					   mlx5_flow_tunnel_grp2tbl_remove_cb);
+					   mlx5_flow_tunnel_grp2tbl_remove_cb,
+					   mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(ipool, id);
 		return NULL;
 	}
-	tunnel->groups->ctx = priv->sh;
 	/* initiate new PMD tunnel */
 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
 	tunnel->tunnel_id = id;
@@ -8666,15 +8735,17 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES), 0,
-					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
+					 rte_align32pow2(MLX5_MAX_TABLES),
+					 false, true, sh,
+					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-					 mlx5_flow_tunnel_grp2tbl_remove_cb);
+					 mlx5_flow_tunnel_grp2tbl_remove_cb,
+					 mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
 	}
-	thub->groups->ctx = sh;
 	sh->tunnel_hub = thub;
 
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ce4d205e86..ab4e8c5c4f 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -480,7 +480,7 @@ struct mlx5_flow_dv_matcher {
 
 /* Encap/decap resource structure. */
 struct mlx5_flow_dv_encap_decap_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/* Pointer to next element. */
 	uint32_t refcnt; /**< Reference counter. */
 	void *action;
@@ -495,7 +495,7 @@ struct mlx5_flow_dv_encap_decap_resource {
 
 /* Tag resource structure. */
 struct mlx5_flow_dv_tag_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry for tag resource, tag value as the key. */
 	void *action;
 	/**< Tag action object. */
@@ -519,7 +519,7 @@ struct mlx5_flow_dv_tag_resource {
 
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
@@ -569,7 +569,7 @@ struct mlx5_flow_mreg_copy_resource {
 	 *  - Key is 32/64-bit MARK action ID.
 	 *  - MUST be the first entry.
 	 */
-	struct mlx5_hlist_entry hlist_ent;
+	struct mlx5_list_entry hlist_ent;
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
 	uint32_t idx;
@@ -586,7 +586,7 @@ struct mlx5_flow_tbl_tunnel_prm {
 
 /* Table data structure of the hash organization. */
 struct mlx5_flow_tbl_data_entry {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
@@ -926,7 +926,7 @@ struct mlx5_flow_tunnel_hub {
 
 /* convert jump group to flow table ID in tunnel rules */
 struct tunnel_tbl_entry {
-	struct mlx5_hlist_entry hash;
+	struct mlx5_list_entry hash;
 	uint32_t flow_table;
 	uint32_t tunnel_id;
 	uint32_t group;
@@ -1573,110 +1573,105 @@ int mlx5_action_handle_flush(struct rte_eth_dev *dev);
 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id);
 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh);
 
-/* Hash list callbacks for flow tables: */
-struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *entry_ctx);
-int flow_dv_tbl_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tbl_create_cb(void *tool_ctx, void *entry_ctx);
+int flow_dv_tbl_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
+void flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tbl_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *entry_ctx);
+void flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
 struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 		uint32_t table_level, uint8_t egress, uint8_t transfer,
 		bool external, const struct mlx5_flow_tunnel *tunnel,
 		uint32_t group_id, uint8_t dummy,
 		uint32_t table_id, struct rte_flow_error *error);
 
-struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *cb_ctx);
-int flow_dv_tag_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx);
+int flow_dv_tag_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
-
-int flow_dv_modify_match_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry,
-			    uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
-						  uint64_t key, void *ctx);
-void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry);
-
-struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
-						uint64_t key, void *ctx);
-int flow_dv_mreg_match_cb(struct mlx5_hlist *list,
-			  struct mlx5_hlist_entry *entry, uint64_t key,
+void flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tag_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *cb_ctx);
+void flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_modify_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_modify_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_modify_clone_cb(void *tool_ctx,
+						struct mlx5_list_entry *oentry,
+						void *ctx);
+void flow_dv_modify_clone_free_cb(void *tool_ctx,
+				  struct mlx5_list_entry *entry);
+
+struct mlx5_list_entry *flow_dv_mreg_create_cb(void *tool_ctx, void *ctx);
+int flow_dv_mreg_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			  void *cb_ctx);
-void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry);
-
-int flow_dv_encap_decap_match_cb(struct mlx5_hlist *list,
-				 struct mlx5_hlist_entry *entry,
-				 uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-				uint64_t key, void *cb_ctx);
-void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry);
-
-int flow_dv_matcher_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *ctx);
-struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
+void flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_mreg_clone_cb(void *tool_ctx,
+					      struct mlx5_list_entry *entry,
+					      void *ctx);
+void flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
 
-int flow_dv_port_id_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused,
+int flow_dv_encap_decap_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 				 void *cb_ctx);
-void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused);
-int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
-						 struct mlx5_list_entry *entry,
-						 void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_sample_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
+struct mlx5_list_entry *flow_dv_encap_decap_create_cb(void *tool_ctx,
+						      void *cb_ctx);
+void flow_dv_encap_decap_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_encap_decap_clone_cb(void *tool_ctx,
 						  struct mlx5_list_entry *entry,
 						  void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+void flow_dv_encap_decap_clone_free_cb(void *tool_ctx,
+				       struct mlx5_list_entry *entry);
+
+int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_matcher_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(void *tool_ctx,
+				     struct mlx5_list_entry *entry __rte_unused,
+				     void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry __rte_unused);
+
+int flow_dv_push_vlan_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			       void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(void *tool_ctx,
+						    void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(void *tool_ctx,
+				     struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_sample_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_dest_array_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+
+int flow_dv_dest_array_match_cb(void *tool_ctx,	struct mlx5_list_entry *entry,
+				void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(void *tool_ctx,
+						     void *cb_ctx);
+void flow_dv_dest_array_remove_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(void *tool_ctx,
+				      struct mlx5_list_entry *entry);
+
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index d588e6fd37..dbe98823bf 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3580,25 +3580,9 @@ flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
 	return 0;
 }
 
-/**
- * Match encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
 int
-flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
-			     struct mlx5_hlist_entry *entry,
-			     uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3617,25 +3601,10 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 	return -1;
 }
 
-/**
- * Allocate encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
-struct mlx5_hlist_entry *
-flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-			      uint64_t key __rte_unused,
-			      void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3673,6 +3642,38 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			     void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	uint32_t idx;
+
+	cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+					   &idx);
+	if (!cache_resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(cache_resource, oentry, sizeof(*cache_resource));
+	cache_resource->idx = idx;
+	return &cache_resource->entry;
+}
+
+void
+flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *res =
+				       container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
+}
+
 /**
  * Find existing encap/decap resource or create and register a new one.
  *
@@ -3697,7 +3698,7 @@ flow_dv_encap_decap_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	union {
 		struct {
 			uint32_t ft_type:8;
@@ -3774,23 +3775,21 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->port_id != res->port_id;
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
@@ -3821,11 +3820,11 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_clone_cb(void *tool_ctx,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
@@ -3843,12 +3842,11 @@ flow_dv_port_id_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
@@ -3893,23 +3891,21 @@ flow_dv_port_id_action_resource_register
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
+			   struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
@@ -3946,11 +3942,11 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_clone_cb(void *tool_ctx,
+			   struct mlx5_list_entry *entry __rte_unused,
+			   void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx;
@@ -3968,12 +3964,11 @@ flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
@@ -5294,30 +5289,14 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
 	return ret;
 }
 
-/**
- * Match modify-header resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx
- *   Pointer to new modify-header resource.
- *
- * @return
- *   0 on matching, non-zero otherwise.
- */
 int
-flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
-			struct mlx5_hlist_entry *entry,
-			uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	struct mlx5_flow_dv_modify_hdr_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
 
 	key_len += ref->actions_num * sizeof(ref->actions[0]);
@@ -5325,11 +5304,10 @@ flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
-			 void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
@@ -5368,6 +5346,33 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
 	return &entry->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *oentry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *entry;
+	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+
+	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	return &entry->entry;
+}
+
+void
+flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 /**
  * Validate the sample action.
  *
@@ -5639,7 +5644,7 @@ flow_dv_modify_hdr_resource_register
 	uint32_t key_len = sizeof(*resource) -
 			   offsetof(typeof(*resource), ft_type) +
 			   resource->actions_num * sizeof(resource->actions[0]);
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
 		.data = resource,
@@ -9915,7 +9920,7 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 }
 
 static struct mlx5_list_entry *
-flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -9938,22 +9943,22 @@ flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 static void
-flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
-	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data;
+	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
 	struct rte_flow_error *error = ctx->error;
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
 	struct mlx5_flow_tbl_resource *tbl;
 	void *domain;
 	uint32_t idx = 0;
@@ -10010,7 +10015,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
 					      flow_dv_matcher_remove_cb,
@@ -10030,13 +10035,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 }
 
 int
-flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key64,
-		     void *cb_ctx __rte_unused)
+flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
 		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
 
 	return tbl_data->level != key.level ||
 	       tbl_data->id != key.id ||
@@ -10045,6 +10050,39 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
 	       tbl_data->is_egress != !!key.is_egress;
 }
 
+struct mlx5_list_entry *
+flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data;
+	struct rte_flow_error *error = ctx->error;
+	uint32_t idx = 0;
+
+	tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
+	if (!tbl_data) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate flow table data entry");
+		return NULL;
+	}
+	memcpy(tbl_data, oentry, sizeof(*tbl_data));
+	tbl_data->idx = idx;
+	return &tbl_data->entry;
+}
+
+void
+flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data =
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
+}
+
 /**
  * Get a flow table.
  *
@@ -10095,9 +10133,10 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = &tt_prm,
+		.data = &table_key.v64,
+		.data2 = &tt_prm,
 	};
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 
 	entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
@@ -10116,12 +10155,11 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
-		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
 
 	MLX5_ASSERT(entry && sh);
 	if (tbl_data->jump.action)
@@ -10129,7 +10167,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 	if (tbl_data->tbl.obj)
 		mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
 	if (tbl_data->tunnel_offload && tbl_data->external) {
-		struct mlx5_hlist_entry *he;
+		struct mlx5_list_entry *he;
 		struct mlx5_hlist *tunnel_grp_hash;
 		struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
 		union tunnel_tbl_key tunnel_key = {
@@ -10138,11 +10176,14 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			.group = tbl_data->group_id
 		};
 		uint32_t table_level = tbl_data->level;
+		struct mlx5_flow_cb_ctx ctx = {
+			.data = (void *)&tunnel_key.val,
+		};
 
 		tunnel_grp_hash = tbl_data->tunnel ?
 					tbl_data->tunnel->groups :
 					thub->groups;
-		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
+		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
 		if (he)
 			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
@@ -10181,7 +10222,7 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10196,11 +10237,9 @@ flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_matcher_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
 	struct mlx5_flow_dv_matcher *resource;
@@ -10297,29 +10336,29 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 	return 0;
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
+struct mlx5_list_entry *
+flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct rte_flow_error *error = ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_tag_resource *entry;
 	uint32_t idx = 0;
 	int ret;
 
 	entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
 	if (!entry) {
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
 	entry->idx = idx;
-	entry->tag_id = key;
-	ret = mlx5_flow_os_create_flow_action_tag(key,
+	entry->tag_id = *(uint32_t *)(ctx->data);
+	ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
 						  &entry->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create action");
 		return NULL;
@@ -10328,14 +10367,45 @@ flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
 }
 
 int
-flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key,
-		     void *cb_ctx __rte_unused)
+flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_tag_resource *tag =
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+
+	return *(uint32_t *)(ctx->data) != tag->tag_id;
+}
+
+struct mlx5_list_entry *
+flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		     void *cb_ctx)
 {
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_tag_resource *entry;
+	uint32_t idx = 0;
+
+	entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate tag resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry));
+	entry->idx = idx;
+	return &entry->entry;
+}
+
+void
+flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
-	return key != tag->tag_id;
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
 }
 
 /**
@@ -10362,9 +10432,13 @@ flow_dv_tag_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_dv_tag_resource *resource;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_cb_ctx ctx = {
+					.error = error,
+					.data = &tag_be24,
+					};
 
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
+	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
@@ -10376,12 +10450,11 @@ flow_dv_tag_resource_register
 }
 
 void
-flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
 	MLX5_ASSERT(tag && sh && tag->action);
 	claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
@@ -10696,7 +10769,7 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
 			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10725,9 +10798,7 @@ flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -10814,7 +10885,7 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
@@ -10840,16 +10911,15 @@ flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			resource->idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
 }
 
 /**
@@ -10892,14 +10962,14 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
 			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_dv_dest_array_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
 	if (ctx_resource->num_of_dest == resource->num_of_dest &&
@@ -10921,9 +10991,7 @@ flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11028,9 +11096,9 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
+			    struct mlx5_list_entry *entry __rte_unused,
+			    void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11056,8 +11124,8 @@ flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
+				 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
 			container_of(entry, typeof(*resource), entry);
@@ -13531,7 +13599,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
 			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_matcher *resource = container_of(entry,
@@ -13568,19 +13636,10 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Release encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- */
 void
-flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry)
+flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
 				       container_of(entry, typeof(*res), entry);
 
@@ -13640,8 +13699,8 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused,
-			 struct mlx5_hlist_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
@@ -13673,10 +13732,9 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
 				  container_of(entry, typeof(*resource), entry);
 
@@ -13730,10 +13788,9 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
 			container_of(entry, typeof(*resource), entry);
 
@@ -13802,7 +13859,7 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
@@ -13850,7 +13907,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 5450ddd388..3f2b99fb65 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index aa9e973d10..7893b3edd4 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,25 +2093,10 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Match an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   0 if match, none zero if not match.
- */
 int
-mlx5_hrxq_match_cb(struct mlx5_list *list,
-		   struct mlx5_list_entry *entry,
-		   void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2251,10 +2236,9 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
 	__mlx5_hrxq_remove(dev, hrxq);
@@ -2305,25 +2289,10 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
 	return NULL;
 }
 
-/**
- * Create an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   queue entry on success, NULL otherwise.
- */
 struct mlx5_list_entry *
-mlx5_hrxq_create_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry __rte_unused,
-		    void *cb_ctx)
+mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq;
@@ -2333,11 +2302,10 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-mlx5_hrxq_clone_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry,
+mlx5_hrxq_clone_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		    void *cb_ctx __rte_unused)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	uint32_t hrxq_idx = 0;
@@ -2351,10 +2319,9 @@ mlx5_hrxq_clone_cb(struct mlx5_list *list,
 }
 
 void
-mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index a4526444f9..94abe79860 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -8,257 +8,6 @@
 
 #include "mlx5_utils.h"
 
-
-/********************* mlx5 list ************************/
-
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
-{
-	struct mlx5_list *list;
-	int i;
-
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return NULL;
-	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
-	return list;
-}
-
-static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
-{
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
-	uint32_t ret;
-
-	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
-			if (reuse) {
-				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_RELAXED) - 1;
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
-					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
-				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_RELAXED);
-			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
-				return entry;
-			if (reuse && ret == 0)
-				entry->ref_cnt--; /* Invalid entry. */
-		}
-		entry = LIST_NEXT(entry, next);
-	}
-	return NULL;
-}
-
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry = NULL;
-	int i;
-
-	rte_rwlock_read_lock(&list->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
-		if (entry)
-			break;
-	}
-	rte_rwlock_read_unlock(&list->lock);
-	return entry;
-}
-
-static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
-		       struct mlx5_list_entry *gentry, void *ctx)
-{
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
-
-	if (unlikely(!lentry))
-		return NULL;
-	lentry->ref_cnt = 1u;
-	lentry->gentry = gentry;
-	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
-	return lentry;
-}
-
-static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
-{
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
-	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
-	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
-					       __ATOMIC_RELAXED);
-
-	while (inv_cnt != 0 && entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
-			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
-			inv_cnt--;
-		}
-		entry = nentry;
-	}
-}
-
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry, *local_entry;
-	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
-	MLX5_ASSERT(list);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
-	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
-	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
-	if (local_entry)
-		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
-		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
-	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
-	if (unlikely(!entry))
-		return NULL;
-	local_entry = list->cb_clone(list, entry, ctx);
-	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
-		return NULL;
-	}
-	entry->ref_cnt = 1u;
-	local_entry->ref_cnt = 1u;
-	local_entry->gentry = entry;
-	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
-	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
-							       RTE_MAX_LCORE,
-							       ctx, true);
-
-		if (unlikely(oentry)) {
-			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
-		}
-	}
-	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
-		(void *)entry, entry->ref_cnt);
-	return local_entry;
-}
-
-int
-mlx5_list_unregister(struct mlx5_list *list,
-		      struct mlx5_list_entry *entry)
-{
-	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
-
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
-	if (entry->lcore_idx == (uint32_t)lcore_idx) {
-		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
-	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
-				   __ATOMIC_RELAXED);
-	} else {
-		return 0;
-	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	rte_rwlock_write_lock(&list->lock);
-	if (likely(gentry->ref_cnt == 0)) {
-		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
-		return 0;
-	}
-	rte_rwlock_write_unlock(&list->lock);
-	return 1;
-}
-
-void
-mlx5_list_destroy(struct mlx5_list *list)
-{
-	struct mlx5_list_entry *entry;
-	int i;
-
-	MLX5_ASSERT(list);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
-			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
-					(void *)entry);
-			} else {
-				list->cb_clone_free(list, entry);
-			}
-		}
-	}
-	mlx5_free(list);
-}
-
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list)
-{
-	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
-}
-
 /********************* Indexed pool **********************/
 
 static inline void
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 0bf2f5f5ca..7d9b64c877 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -296,203 +296,6 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ mlx5 list *****************************/
-
-/** Maximum size of string for naming. */
-#define MLX5_NAME_SIZE			32
-
-struct mlx5_list;
-
-/**
- * Structure of the entry in the mlx5 list, user should define its own struct
- * that contains this in order to store the data.
- */
-struct mlx5_list_entry {
-	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* 0 means, entry is invalid. */
-	uint32_t lcore_idx;
-	struct mlx5_list_entry *gentry;
-};
-
-struct mlx5_list_cache {
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
-	uint32_t inv_cnt; /* Invalid entries counter. */
-} __rte_cache_aligned;
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
-				   struct mlx5_list_entry *entry, void *ctx);
-
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
-
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined mlx5 list entry creation.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer of entry on success, NULL otherwise.
- */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
-
-/**
- * Linked mlx5 list structure.
- *
- * Entry in mlx5 list could be reused if entry already exists,
- * reference count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- *
- * Linked list is designed for limited number of entries,
- * read mostly, less modification.
- *
- * For huge amount of entries, please consider hash list.
- *
- */
-struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
-	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-};
-
-/**
- * Create a mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param name
- *   Name of the mlx5 list.
- * @param ctx
- *   Pointer to the list context data.
- * @param cb_create
- *   Callback function for entry create.
- * @param cb_match
- *   Callback function for entry match.
- * @param cb_remove
- *   Callback function for entry remove.
- * @return
- *   List pointer on success, otherwise NULL.
- */
-struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
-				   mlx5_list_create_cb cb_create,
-				   mlx5_list_match_cb cb_match,
-				   mlx5_list_remove_cb cb_remove,
-				   mlx5_list_clone_cb cb_clone,
-				   mlx5_list_clone_free_cb cb_clone_free);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param list
- *   Pointer to the mlx5 list.
- * @param ctx
- *   Common context parameter used by entry callback function.
- *
- * @return
- *   Pointer of the list entry if found, NULL otherwise.
- */
-struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
-					   void *ctx);
-
-/**
- * Reuse or create an entry to the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param ctx
- *   Common context parameter used by callback function.
- *
- * @return
- *   registered entry on success, NULL otherwise
- */
-struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
-					     void *ctx);
-
-/**
- * Remove an entry from the mlx5 list.
- *
- * User should guarantee the validity of the entry.
- *
- * @param list
- *   Pointer to the hast list.
- * @param entry
- *   Entry to be removed from the mlx5 list table.
- * @return
- *   0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_list_unregister(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry);
-
-/**
- * Destroy the mlx5 list.
- *
- * @param list
- *   Pointer to the mlx5 list.
- */
-void mlx5_list_destroy(struct mlx5_list *list);
-
-/**
- * Get entry number from the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list.
- * @return
- *   mlx5 list entry number.
- */
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list);
-
 /********************************* indexed pool *************************/
 
 /**
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 97a8f04e39..af2f684648 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -608,7 +608,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v3 13/22] net/mlx5: move modify header allocator to ipool

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Modify header actions are allocated by mlx5_malloc which has a big
overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has
its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in
the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has
better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header
handler.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0e80408511..713accf675 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -801,6 +801,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -814,6 +815,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index f3768ee028..c7239e1137 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1123,6 +1136,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ab4e8c5c4f..4552aaa803 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -504,23 +504,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index dbe98823bf..e702b78358 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5304,6 +5304,45 @@ flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
+static struct mlx5_indexed_pool *
+flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
+{
+	struct mlx5_indexed_pool *ipool = __atomic_load_n
+				     (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
+
+	if (!ipool) {
+		struct mlx5_indexed_pool *expected = NULL;
+		struct mlx5_indexed_pool_config cfg =
+		    (struct mlx5_indexed_pool_config) {
+		       .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
+								   (index + 1) *
+					   sizeof(struct mlx5_modification_cmd),
+		       .trunk_size = 64,
+		       .grow_trunk = 3,
+		       .grow_shift = 2,
+		       .need_lock = 1,
+		       .release_mem_en = 1,
+		       .malloc = mlx5_malloc,
+		       .free = mlx5_free,
+		       .type = "mlx5_modify_action_resource",
+		};
+
+		cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
+		ipool = mlx5_ipool_create(&cfg);
+		if (!ipool)
+			return NULL;
+		if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
+						 &expected, ipool, false,
+						 __ATOMIC_SEQ_CST,
+						 __ATOMIC_SEQ_CST)) {
+			mlx5_ipool_destroy(ipool);
+			ipool = __atomic_load_n(&sh->mdh_ipools[index],
+						__ATOMIC_SEQ_CST);
+		}
+	}
+	return ipool;
+}
+
 struct mlx5_list_entry *
 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
@@ -5312,12 +5351,20 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
+							  ref->actions_num - 1);
 	int ret;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
+	uint32_t idx;
 
-	entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
-			    SOCKET_ID_ANY);
+	if (unlikely(!ipool)) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL, "cannot allocate modify ipool");
+		return NULL;
+	}
+	entry = mlx5_ipool_zmalloc(ipool, &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5337,25 +5384,29 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 					(sh->ctx, ns, entry,
 					 data_len, &entry->action);
 	if (ret) {
-		mlx5_free(entry);
+		mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create modification action");
 		return NULL;
 	}
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 struct mlx5_list_entry *
-flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
-			struct mlx5_list_entry *oentry, void *cb_ctx)
+flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			void *cb_ctx)
 {
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+	uint32_t idx;
 
-	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
+				  &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5363,14 +5414,18 @@ flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
 		return NULL;
 	}
 	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 void
-flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	mlx5_free(entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *res =
+		container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
@@ -13699,14 +13754,14 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
-			 struct mlx5_list_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
-	mlx5_free(entry);
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v3 14/22] net/mlx5: adjust the hash bucket size

[Suanming Mou]

With the new per core optimization to the list, the hash bucket size
can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index cf573a9a4d..a82dc4db00 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 713accf675..8fb7f4442d 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -373,7 +373,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 906aa43c5a..ca67ce8213 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 7bd45d3895..15a895ea48 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -8625,7 +8625,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8734,8 +8734,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-- 
2.25.1

[dpdk-dev] [PATCH v3 15/22] common/mlx5: allocate cache list memory individually

[Suanming Mou]

Currently, the list's local cache instance memory is allocated with
the list. As the local cache instance array size is RTE_MAX_LCORE,
most of the cases the system will only have very limited cores.
allocate the instance memory individually per core will be more
economic to the memory.

This commit changes the instance array to pointer array, allocate
the local cache memory only when the core is to be used.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 4e385c616a..f75b1cb0da 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 61b30a45ca..979dfafad4 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-- 
2.25.1

[dpdk-dev] [PATCH v3 16/22] net/mlx5: enable index pool per-core cache

[Suanming Mou]

This commit enables the tag and header modify action index pool
per-core cache in non-reclaim memory mode.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 8fb7f4442d..bf1463c289 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -214,7 +214,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1128,6 +1129,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index c7239e1137..01198b6cc7 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1093,6 +1093,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index e702b78358..f84a2b1a5d 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5321,7 +5321,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
-- 
2.25.1

[dpdk-dev] [PATCH v3 17/22] net/mlx5: optimize hash list table allocate on demand

[Suanming Mou]

Currently, all the hash list tables are allocated during start up.
Since different applications may only use dedicated limited actions,
optimized the hash list table allocate on demand will save initial
memory.

This commit optimizes hash list table allocate on demand.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index a82dc4db00..75324e35d8 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index ca67ce8213..fe86bb40d3 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index f84a2b1a5d..bb70e8557f 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -310,6 +310,41 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 	}
 }
 
+static inline struct mlx5_hlist *
+flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
+		     const char *name, uint32_t size, bool direct_key,
+		     bool lcores_share, void *ctx,
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove,
+		     mlx5_list_clone_cb cb_clone,
+		     mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_hlist *hl;
+	struct mlx5_hlist *expected = NULL;
+	char s[MLX5_NAME_SIZE];
+
+	hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	if (likely(hl))
+		return hl;
+	snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
+	hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
+			ctx, cb_create, cb_match, cb_remove, cb_clone,
+			cb_clone_free);
+	if (!hl) {
+		DRV_LOG(ERR, "%s hash creation failed", name);
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
+					 __ATOMIC_SEQ_CST,
+					 __ATOMIC_SEQ_CST)) {
+		mlx5_hlist_destroy(hl);
+		hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	}
+	return hl;
+}
+
 /* Update VLAN's VID/PCP based on input rte_flow_action.
  *
  * @param[in] action
@@ -3724,8 +3759,20 @@ flow_dv_encap_decap_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *encaps_decaps;
 	uint64_t key64;
 
+	encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
+				"encaps_decaps",
+				MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
+				true, true, sh,
+				flow_dv_encap_decap_create_cb,
+				flow_dv_encap_decap_match_cb,
+				flow_dv_encap_decap_remove_cb,
+				flow_dv_encap_decap_clone_cb,
+				flow_dv_encap_decap_clone_free_cb);
+	if (unlikely(!encaps_decaps))
+		return -rte_errno;
 	resource->flags = dev_flow->dv.group ? 0 : 1;
 	key64 =  __rte_raw_cksum(&encap_decap_key.v32,
 				 sizeof(encap_decap_key.v32), 0);
@@ -3733,7 +3780,7 @@ flow_dv_encap_decap_resource_register
 	    MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
 	    resource->size)
 		key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
-	entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
+	entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -5705,8 +5752,20 @@ flow_dv_modify_hdr_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *modify_cmds;
 	uint64_t key64;
 
+	modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
+				"hdr_modify",
+				MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
+				true, false, sh,
+				flow_dv_modify_create_cb,
+				flow_dv_modify_match_cb,
+				flow_dv_modify_remove_cb,
+				flow_dv_modify_clone_cb,
+				flow_dv_modify_clone_free_cb);
+	if (unlikely(!modify_cmds))
+		return -rte_errno;
 	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
 								resource->root))
@@ -5714,7 +5773,7 @@ flow_dv_modify_hdr_resource_register
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
 	key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
-	entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
+	entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10493,8 +10552,20 @@ flow_dv_tag_resource_register
 					.error = error,
 					.data = &tag_be24,
 					};
-
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
+	struct mlx5_hlist *tag_table;
+
+	tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
+				      "tags",
+				      MLX5_TAGS_HLIST_ARRAY_SIZE,
+				      false, false, priv->sh,
+				      flow_dv_tag_create_cb,
+				      flow_dv_tag_match_cb,
+				      flow_dv_tag_remove_cb,
+				      flow_dv_tag_clone_cb,
+				      flow_dv_tag_clone_free_cb);
+	if (unlikely(!tag_table))
+		return -rte_errno;
+	entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index af2f684648..0e6f7003b0 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
-- 
2.25.1

[dpdk-dev] [PATCH v3 18/22] common/mlx5: optimize cache list object memory

[Suanming Mou]

Currently, hash list uses the cache list as bucket list. The list
in the buckets have the same name, ctx and callbacks. This wastes
the memory.

This commit abstracts all the name, ctx and callback members in the
list to a constant struct and others to the inconstant struct, uses
the wrapper functions to satisfy both hash list and cache list can
set the list constant and inconstant struct individually.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index f75b1cb0da..858c8d8164 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 979dfafad4..9e8ebe772a 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -80,6 +80,32 @@ typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
 typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
 						       void *ctx);
 
+/**
+ * Linked mlx5 list constant object.
+ */
+struct mlx5_list_const {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	void *ctx; /* user objects target to callback. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	/**< entry clone free callback. */
+};
+
+/**
+ * Linked mlx5 list inconstant data.
+ */
+struct mlx5_list_inconst {
+	rte_rwlock_t lock; /* read/write lock. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+};
+
 /**
  * Linked mlx5 list structure.
  *
@@ -96,19 +122,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
  *
  */
 struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	void *ctx; /* user objects target to callback. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-	volatile uint32_t gen_cnt; /* List modification may update it. */
-	volatile uint32_t count; /* number of entries in list. */
-	rte_rwlock_t lock; /* read/write lock. */
+	struct mlx5_list_const l_const;
+	struct mlx5_list_inconst l_inconst;
 };
 
 /**
@@ -214,7 +229,7 @@ mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_list l;
+	struct mlx5_list_inconst l;
 } __rte_cache_aligned;
 
 /**
@@ -226,7 +241,7 @@ struct mlx5_hlist {
 	uint32_t mask; /* A mask for the bucket index range. */
 	uint8_t flags;
 	bool direct_key; /* Whether to use the key directly as hash index. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
+	struct mlx5_list_const l_const; /* List constant data. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v3 19/22] net/mlx5: change memory release configuration

[Suanming Mou]

This commit changes the index pool memory release configuration
to 0 when memory reclaim mode is not required.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index bf1463c289..6b7225e55d 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -797,6 +797,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
-- 
2.25.1

[dpdk-dev] [PATCH v3 20/22] net/mlx5: support index pool none local core operations

[Suanming Mou]

This commit supports the index pool none local core operations with
an extra cache.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 94abe79860..c34d6d62a8 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -114,6 +114,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->nlcore_lock);
 	return pool;
 }
 
@@ -354,20 +355,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -378,15 +373,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -399,29 +406,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -464,6 +482,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -643,7 +676,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -712,7 +745,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 7d9b64c877..060c52f022 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t nlcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
-- 
2.25.1

[dpdk-dev] [PATCH v3 21/22] net/mlx5: support list none local core operations

[Suanming Mou]

This commit supports the list none local core operations with
an extra sub-list.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 858c8d8164..d58d0d08ab 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.nlcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.nlcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.nlcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.nlcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.nlcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.nlcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.nlcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.nlcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.nlcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.nlcore_lock);
+	return ret;
 }
 
 void
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 9e8ebe772a..95106afc5e 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t nlcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v3 22/22] net/mlx5: optimize Rx queue match

[Suanming Mou]

As hrxq struct has the indirect table pointer, while matching the
hrxq, better to use the hrxq indirect table instead of searching
from the list.

This commit optimizes the hrxq indirect table matching.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 7893b3edd4..23685d7654 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v4 00/26] net/mlx5: insertion rate optimization

[Suanming Mou]

This patch series optimize the flow insertion rate with adding
local cache to index pool and list.

For object which wants efficient index allocate and free, local
cache will be very helpful.

For index pool, two level cache is added, one as local and another
as global. The global cache is able to save all the allocated
index. That means all the allocated index will not be freed. Once
the local cache is full, the extra index will be flushed to the
global cache. Once local cache is empty, first try to fetch more
index from global, if global is still empty, allocate new trunk
and more index.

For list, sub local core list is introduced. The allocated objects
will be added and released only from local list without any locks.
Only the objects need to be shared will be synced from global list.

---

v4:
   - split the list utility to commom patch
   - fix one ipool crash issue in mlx5_ipool_get_cache() function.
   - reorganize the common and mlx5 patches.
   - split the doc update to an individual patch.
   - update and rebase to the latest version.

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
  net/mlx5: optimize modify header action memory
  net/mlx5: remove cache term from the list utility
  net/mlx5: add per lcore cache to the list utility
  net/mlx5: minimize list critical sections
  net/mlx5: manage list cache entries release
  net/mlx5: relax the list utility atomic operations
  net/mlx5: allocate list memory by the create API
  common/mlx5: add per-lcore cache to hash list utility
  net/mlx5: move modify header allocator to ipool

Suanming Mou (17):
  net/mlx5: allow limiting the index pool maximum index
  net/mlx5: add indexed pool local cache
  net/mlx5: add index pool foreach define
  net/mlx5: support index pool non-lcore operations
  net/mlx5: replace flow list with index pool
  common/mlx5: move list utility to common
  common/mlx5: add list lcore share
  common/mlx5: call list callbacks with context
  common/mlx5: allocate cache list memory individually
  common/mlx5: optimize cache list object memory
  common/mlx5: support list non-lcore operations
  net/mlx5: adjust the hash bucket size
  net/mlx5: enable index pool per-core cache
  net/mlx5: optimize hash list table allocate on demand
  net/mlx5: change memory release configuration
  net/mlx5: optimize Rx queue match
  doc: add mlx5 multiple-thread flow insertion optimization

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    6 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  294 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  305 +++---
 drivers/net/mlx5/mlx5_flow.h            |  209 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1203 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  627 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2545 insertions(+), 1411 deletions(-)

-- 
2.25.1

[dpdk-dev] [PATCH v4 01/26] net/mlx5: allow limiting the index pool maximum index

[Suanming Mou]

Some ipool instances in the driver are used as ID\index allocator and
added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value.
The ipool will ensure that no index bigger than the maximum value is
provided.

Use this configuration in ID allocator cases instead of the current
logics. This patch add the maximum ID configurable for the index pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 18fe23e4fb..bf2b2ebc72 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index b54517c6df..15870e14c2 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
-- 
2.25.1

[dpdk-dev] [PATCH v4 02/26] net/mlx5: add indexed pool local cache

[Suanming Mou]

For object which wants efficient index allocate and free, local
cache will be very helpful.

Two level cache is introduced to allocate and free the index more
efficient. One as local and the other as global. The global cache
is able to save all the allocated index. That means all the allocated
index will not be freed. Once the local cache is full, the extra
index will be flushed to the global cache. Once local cache is empty,
first try to fetch more index from global, if global is still empty,
allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index bf2b2ebc72..215024632d 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 15870e14c2..0469062695 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -209,6 +209,11 @@ struct mlx5_indexed_pool_config {
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
 	uint32_t max_idx; /* The maximum index can be allocated. */
+	uint32_t per_core_cache;
+	/*
+	 * Cache entry number per core for performance. Should not be
+	 * set with release_mem_en.
+	 */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
@@ -225,14 +230,39 @@ struct mlx5_indexed_trunk {
 	uint8_t data[] __rte_cache_aligned; /* Entry data start. */
 };
 
+struct mlx5_indexed_cache {
+	struct mlx5_indexed_trunk **trunks;
+	volatile uint32_t n_trunk_valid; /* Trunks allocated. */
+	uint32_t n_trunk; /* Trunk pointer array size. */
+	uint32_t ref_cnt;
+	uint32_t len;
+	uint32_t idx[];
+};
+
+struct mlx5_ipool_per_lcore {
+	struct mlx5_indexed_cache *lc;
+	uint32_t len; /**< Current cache count. */
+	uint32_t idx[]; /**< Cache objects. */
+};
+
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
-	rte_spinlock_t lock; /* Pool lock for multiple thread usage. */
-	uint32_t n_trunk_valid; /* Trunks allocated. */
-	uint32_t n_trunk; /* Trunk pointer array size. */
+	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
 	/* Dim of trunk pointer array. */
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t free_list; /* Index to first free trunk. */
+	union {
+		struct {
+			uint32_t n_trunk_valid; /* Trunks allocated. */
+			uint32_t n_trunk; /* Trunk pointer array size. */
+			struct mlx5_indexed_trunk **trunks;
+			uint32_t free_list; /* Index to first free trunk. */
+		};
+		struct {
+			struct mlx5_indexed_cache *gc;
+			/* Global cache. */
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			/* Local cache. */
+		};
+	};
 #ifdef POOL_DEBUG
 	uint32_t n_entry;
 	uint32_t trunk_new;
@@ -542,6 +572,30 @@ int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool);
  */
 void mlx5_ipool_dump(struct mlx5_indexed_pool *pool);
 
+/**
+ * This function flushes all the cache index back to pool trunk.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ *
+ */
+
+void mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool);
+
+/**
+ * This function gets the available entry from pos.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ * @param pos
+ *   Pointer to the index position start from.
+ *
+ * @return
+ *  - Pointer to the next available entry.
+ *
+ */
+void *mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos);
+
 /**
  * This function allocates new empty Three-level table.
  *
-- 
2.25.1

[dpdk-dev] [PATCH v4 03/26] net/mlx5: add index pool foreach define

[Suanming Mou]

In some cases, application may want to know all the allocated
index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach
operation.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 96 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 +++
 2 files changed, 104 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 215024632d..32f8d65073 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -529,6 +529,16 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -839,6 +849,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 0469062695..737dd7052d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
-- 
2.25.1

[dpdk-dev] [PATCH v4 04/26] net/mlx5: support index pool non-lcore operations

[Suanming Mou]

This commit supports the index pool non-lcore operations with
an extra cache and lcore lock.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 32f8d65073..f9557c09ff 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -275,6 +275,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->lcore_lock);
 	return pool;
 }
 
@@ -515,20 +516,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -549,15 +544,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -570,29 +577,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -635,6 +653,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -814,7 +847,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -883,7 +916,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 737dd7052d..a509b0a4eb 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t lcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
-- 
2.25.1

[dpdk-dev] [PATCH v4 05/26] net/mlx5: replace flow list with index pool

[Suanming Mou]

The flow list is used to save the create flows and to be used only
when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to
flush all the allocated flows.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 +++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 149 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 126 insertions(+), 110 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index b3f9e392ab..4b050b4f4a 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -823,6 +861,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1566,7 +1605,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1602,6 +1640,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index abd573e8ed..2c99da0fac 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -322,7 +322,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -790,8 +791,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1526,7 +1529,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 0f4b239142..b196fd365f 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1344,7 +1352,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1597,7 +1606,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c27f6197a0..8e3af5bccb 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3095,31 +3095,6 @@ mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
 					 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
 }
 
-/**
- * Release resource related QUEUE/RSS action split.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Flow to release id's from.
- */
-static void
-flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
-			     struct rte_flow *flow)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t handle_idx;
-	struct mlx5_flow_handle *dev_handle;
-
-	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
-		       handle_idx, dev_handle, next)
-		if (dev_handle->split_flow_id &&
-		    !dev_handle->is_meter_flow_id)
-			mlx5_ipool_free(priv->sh->ipool
-					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
-					dev_handle->split_flow_id);
-}
-
 static int
 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
 		   const struct rte_flow_attr *attr __rte_unused,
@@ -3415,7 +3390,6 @@ flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 	const struct mlx5_flow_driver_ops *fops;
 	enum mlx5_flow_drv_type type = flow->drv_type;
 
-	flow_mreg_split_qrss_release(dev, flow);
 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
 	fops = flow_get_drv_ops(type);
 	fops->destroy(dev, flow);
@@ -3998,14 +3972,14 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
 
 /* Declare flow create/destroy prototype in advance. */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action actions[],
 		 bool external, struct rte_flow_error *error);
 
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
@@ -4127,8 +4101,8 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	 * be applied, removed, deleted in ardbitrary order
 	 * by list traversing.
 	 */
-	mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
-					 actions, false, error);
+	mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
+					&attr, items, actions, false, error);
 	if (!mcp_res->rix_flow) {
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
 		return NULL;
@@ -4190,7 +4164,7 @@ flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
-	flow_list_destroy(dev, NULL, mcp_res->rix_flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
 }
 
@@ -6110,7 +6084,7 @@ flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
  *   A flow index on success, 0 otherwise and rte_errno is set.
  */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action original_actions[],
@@ -6178,7 +6152,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 				external, hairpin_flow, error);
 	if (ret < 0)
 		goto error_before_hairpin_split;
-	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
+	flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
 	if (!flow) {
 		rte_errno = ENOMEM;
 		goto error_before_hairpin_split;
@@ -6308,12 +6282,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 		if (ret < 0)
 			goto error;
 	}
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
-			     flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
+	flow->type = type;
 	flow_rxq_flags_set(dev, flow);
 	rte_free(translated_actions);
 	tunnel = flow_tunnel_from_rule(wks->flows);
@@ -6335,7 +6304,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_ipool_get
 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
 			rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
+	mlx5_ipool_free(priv->flows[type], idx);
 	rte_errno = ret; /* Restore rte_errno. */
 	ret = rte_errno;
 	rte_errno = ret;
@@ -6387,10 +6356,9 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
 			.type = RTE_FLOW_ACTION_TYPE_END,
 		},
 	};
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow_error error;
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
 						   &attr, &pattern,
 						   actions, false, &error);
 }
@@ -6442,8 +6410,6 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
 	/*
 	 * If the device is not started yet, it is not allowed to created a
 	 * flow from application. PMD default flows and traffic control flows
@@ -6459,8 +6425,9 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		return NULL;
 	}
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
-				  attr, items, actions, true, error);
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
+						   attr, items, actions,
+						   true, error);
 }
 
 /**
@@ -6468,24 +6435,19 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list. If this parameter NULL,
- *   there is no flow removal from the list. Be noted that as
- *   flow is add to the indexed list, memory of the indexed
- *   list points to maybe changed as flow destroyed.
  * @param[in] flow_idx
  *   Index of flow to destroy.
  */
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW], flow_idx);
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
 
 	if (!flow)
 		return;
+	MLX5_ASSERT(flow->type == type);
 	/*
 	 * Update RX queue flags only if port is started, otherwise it is
 	 * already clean.
@@ -6493,12 +6455,6 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
 	flow_drv_destroy(dev, flow);
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
-			     flow_idx, flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
 	if (flow->tunnel) {
 		struct mlx5_flow_tunnel *tunnel;
 
@@ -6508,7 +6464,7 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
 	flow_mreg_del_copy_action(dev, flow);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
+	mlx5_ipool_free(priv->flows[type], flow_idx);
 }
 
 /**
@@ -6516,18 +6472,21 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list.
+ * @param type
+ *   Flow type to be flushed.
  * @param active
  *   If flushing is called avtively.
  */
 void
-mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
+mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+		     bool active)
 {
-	uint32_t num_flushed = 0;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t num_flushed = 0, fidx = 1;
+	struct rte_flow *flow;
 
-	while (*list) {
-		flow_list_destroy(dev, list, *list);
+	MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
+		flow_list_destroy(dev, type, fidx);
 		num_flushed++;
 	}
 	if (active) {
@@ -6699,18 +6658,19 @@ mlx5_flow_pop_thread_workspace(void)
  * @return the number of flows not released.
  */
 int
-mlx5_flow_verify(struct rte_eth_dev *dev)
+mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
-	uint32_t idx;
-	int ret = 0;
+	uint32_t idx = 0;
+	int ret = 0, i;
 
-	ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
-		      flow, next) {
-		DRV_LOG(DEBUG, "port %u flow %p still referenced",
-			dev->data->port_id, (void *)flow);
-		++ret;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
+			DRV_LOG(DEBUG, "port %u flow %p still referenced",
+				dev->data->port_id, (void *)flow);
+			ret++;
+		}
 	}
 	return ret;
 }
@@ -6730,7 +6690,6 @@ int
 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 			    uint32_t queue)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.egress = 1,
 		.priority = 0,
@@ -6763,8 +6722,8 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 	actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
 	actions[0].conf = &jump;
 	actions[1].type = RTE_FLOW_ACTION_TYPE_END;
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx) {
 		DRV_LOG(DEBUG,
 			"Failed to create ctrl flow: rte_errno(%d),"
@@ -6853,8 +6812,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 		action_rss.types = 0;
 	for (i = 0; i != priv->reta_idx_n; ++i)
 		queue[i] = (*priv->reta_idx)[i];
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx)
 		return -rte_errno;
 	return 0;
@@ -6895,7 +6854,6 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev,
 int
 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	/*
 	 * The LACP matching is done by only using ether type since using
 	 * a multicast dst mac causes kernel to give low priority to this flow.
@@ -6929,8 +6887,9 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 		},
 	};
 	struct rte_flow_error error;
-	uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+					&attr, items, actions,
+					false, &error);
 
 	if (!flow_idx)
 		return -rte_errno;
@@ -6948,9 +6907,8 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
 		  struct rte_flow *flow,
 		  struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+				(uintptr_t)(void *)flow);
 	return 0;
 }
 
@@ -6964,9 +6922,7 @@ int
 mlx5_flow_flush(struct rte_eth_dev *dev,
 		struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
 	return 0;
 }
 
@@ -7017,8 +6973,7 @@ flow_drv_query(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct mlx5_flow_driver_ops *fops;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW],
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 					       flow_idx);
 	enum mlx5_flow_drv_type ftype;
 
@@ -7884,14 +7839,14 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		if (!config->dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
-		flow_idx = flow_list_create(dev, NULL, &attr, items,
-					    actions, false, &error);
-		flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
+					items, actions, false, &error);
+		flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 				      flow_idx);
 		if (!flow)
 			continue;
 		config->flow_mreg_c[n++] = idx;
-		flow_list_destroy(dev, NULL, flow_idx);
+		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
 	}
 	for (; n < MLX5_MREG_C_NUM; ++n)
 		config->flow_mreg_c[n] = REG_NON;
@@ -7935,8 +7890,8 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
 					sh->rx_domain,
 					sh->tx_domain, file);
 	/* dump one */
-	flow = mlx5_ipool_get(priv->sh->ipool
-			[MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
+	flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
+			(uintptr_t)(void *)flow_idx);
 	if (!flow)
 		return -ENOENT;
 
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2f2aa962f9..d9b6acaafd 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -997,9 +997,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 75ef6216ac..eee3926c06 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13881,6 +13881,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ae7fcca229..7cb8920d6b 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index e30b682822..17716b66c9 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
-- 
2.25.1

[dpdk-dev] [PATCH v4 06/26] net/mlx5: optimize modify header action memory

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Define the types of the modify header action fields to be with the
minimum size needed for the optional values range.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_glue.h b/drivers/common/mlx5/linux/mlx5_glue.h
index 840d8cf57f..a186ee577f 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.h
+++ b/drivers/common/mlx5/linux/mlx5_glue.h
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
diff --git a/drivers/net/mlx5/linux/mlx5_flow_os.h b/drivers/net/mlx5/linux/mlx5_flow_os.h
index cee685015b..1926d26410 100644
--- a/drivers/net/mlx5/linux/mlx5_flow_os.h
+++ b/drivers/net/mlx5/linux/mlx5_flow_os.h
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index d9b6acaafd..81c95e0beb 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -523,11 +523,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index eee3926c06..bca8339361 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5000,21 +5000,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5616,10 +5616,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
-- 
2.25.1

[dpdk-dev] [PATCH v4 07/26] net/mlx5: remove cache term from the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The internal mlx5 list tool is used mainly when the list objects need to
be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread
cache management.

To prevent confusing, remove the current "cache" term from the API's
names.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 558 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 454 insertions(+), 460 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 4b050b4f4a..57b0a1c57f 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -500,8 +500,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1704,7 +1704,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1765,7 +1765,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 2c99da0fac..f51be5ace6 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1609,7 +1609,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index b196fd365f..bf1fbb530b 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1221,7 +1221,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1369,7 +1369,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 81c95e0beb..4dec703366 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -467,7 +467,7 @@ struct mlx5_flow_dv_match_params {
 
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
-	struct mlx5_cache_entry entry; /**< Pointer to the next element. */
+	struct mlx5_list_entry entry; /**< Pointer to the next element. */
 	struct mlx5_flow_tbl_resource *tbl;
 	/**< Pointer to the table(group) the matcher associated with. */
 	void *matcher_object; /**< Pointer to DV matcher */
@@ -547,7 +547,7 @@ struct mlx5_flow_dv_jump_tbl_resource {
 
 /* Port ID resource structure. */
 struct mlx5_flow_dv_port_id_action_resource {
-	struct mlx5_cache_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Action object. */
 	uint32_t port_id; /**< Port ID value. */
 	uint32_t idx; /**< Indexed pool memory index. */
@@ -555,7 +555,7 @@ struct mlx5_flow_dv_port_id_action_resource {
 
 /* Push VLAN action resource structure */
 struct mlx5_flow_dv_push_vlan_action_resource {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* Cache entry. */
 	void *action; /**< Action object. */
 	uint8_t ft_type; /**< Flow table type, Rx, Tx or FDB. */
 	rte_be32_t vlan_tag; /**< VLAN tag value. */
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_cache_list matchers;
+	struct mlx5_list matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
@@ -631,7 +631,7 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	union {
 		void *verbs_action; /**< Verbs sample action object. */
 		void **sub_actions; /**< Sample sub-action array. */
@@ -653,7 +653,7 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
@@ -1619,43 +1619,45 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
 
-int flow_dv_matcher_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *ctx);
-struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_port_id_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_sample_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
-			      struct mlx5_cache_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
-				struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_dest_array_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry);
+int flow_dv_matcher_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *ctx);
+void flow_dv_matcher_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
+						 struct mlx5_list_entry *entry,
+						 void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index bca8339361..d19b41c20a 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3601,18 +3601,17 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 			     uint64_t key __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
-
-	cache_resource = container_of(entry,
-				      struct mlx5_flow_dv_encap_decap_resource,
-				      entry);
-	if (resource->reformat_type == cache_resource->reformat_type &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->flags == cache_resource->flags &&
-	    resource->size == cache_resource->size &&
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
+
+	resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
+				entry);
+	if (resource->reformat_type == ctx_resource->reformat_type &&
+	    resource->ft_type == ctx_resource->ft_type &&
+	    resource->flags == ctx_resource->flags &&
+	    resource->size == ctx_resource->size &&
 	    !memcmp((const void *)resource->buf,
-		    (const void *)cache_resource->buf,
+		    (const void *)ctx_resource->buf,
 		    resource->size))
 		return 0;
 	return -1;
@@ -3639,31 +3638,30 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 	uint32_t idx;
 	int ret;
 
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
 	/* Register new encap/decap resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-				       &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
-	cache_resource->idx = idx;
-	ret = mlx5_flow_os_create_flow_action_packet_reformat
-					(sh->ctx, domain, cache_resource,
-					 &cache_resource->action);
+	*resource = *ctx_resource;
+	resource->idx = idx;
+	ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
+							      resource,
+							     &resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3672,7 +3670,7 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 		return NULL;
 	}
 
-	return &cache_resource->entry;
+	return &resource->entry;
 }
 
 /**
@@ -3776,8 +3774,8 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
@@ -3787,30 +3785,30 @@ flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->port_id != res->port_id;
 }
 
-struct mlx5_cache_entry *
-flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_port_id_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate port_id action cache memory");
+				   "cannot allocate port_id action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
 							ref->port_id,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3818,8 +3816,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 				   "cannot create action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3827,8 +3825,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3840,30 +3838,30 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_port_id_action_resource_register
 			(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_port_id_action_resource *resource,
+			 struct mlx5_flow_dv_port_id_action_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->dv.port_id_action = cache;
-	dev_flow->handle->rix_port_id_action = cache->idx;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->dv.port_id_action = resource;
+	dev_flow->handle->rix_port_id_action = resource->idx;
 	return 0;
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
@@ -3873,28 +3871,28 @@ flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
-struct mlx5_cache_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	struct mlx5dv_dr_domain *domain;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate push_vlan action cache memory");
+				   "cannot allocate push_vlan action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
@@ -3902,7 +3900,7 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 	else
 		domain = sh->tx_domain;
 	ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3910,8 +3908,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 				   "cannot create push vlan action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3919,8 +3917,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
  *
  * @param [in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3932,25 +3930,25 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_push_vlan_action_resource_register
 		       (struct rte_eth_dev *dev,
-			struct mlx5_flow_dv_push_vlan_action_resource *resource,
+			struct mlx5_flow_dv_push_vlan_action_resource *ref,
 			struct mlx5_flow *dev_flow,
 			struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
+	resource = container_of(entry, typeof(*resource), entry);
 
-	dev_flow->handle->dvh.rix_push_vlan = cache->idx;
-	dev_flow->dv.push_vlan_res = cache;
+	dev_flow->handle->dvh.rix_push_vlan = resource->idx;
+	dev_flow->dv.push_vlan_res = resource;
 	return 0;
 }
 
@@ -9950,13 +9948,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 			return NULL;
 		}
 	}
-	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache",
+	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh,
-			     flow_dv_matcher_create_cb,
-			     flow_dv_matcher_match_cb,
-			     flow_dv_matcher_remove_cb);
+	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+			 flow_dv_matcher_create_cb,
+			 flow_dv_matcher_match_cb,
+			 flow_dv_matcher_remove_cb);
 	return &tbl_data->entry;
 }
 
@@ -10084,7 +10082,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_cache_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(&tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10112,8 +10110,8 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
@@ -10126,15 +10124,15 @@ flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
 		      (const void *)ref->mask.buf, ref->mask.size);
 }
 
-struct mlx5_cache_entry *
-flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_matcher_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5dv_flow_matcher_attr dv_attr = {
 		.type = IBV_FLOW_ATTR_NORMAL,
 		.match_mask = (void *)&ref->mask,
@@ -10143,29 +10141,30 @@ flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
 							    typeof(*tbl), tbl);
 	int ret;
 
-	cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY);
-	if (!cache) {
+	resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
+			       SOCKET_ID_ANY);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	dv_attr.match_criteria_enable =
-		flow_dv_matcher_enable(cache->mask.buf);
+		flow_dv_matcher_enable(resource->mask.buf);
 	dv_attr.priority = ref->priority;
 	if (tbl->is_egress)
 		dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
 	ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
-					       &cache->matcher_object);
+					       &resource->matcher_object);
 	if (ret) {
-		mlx5_free(cache);
+		mlx5_free(resource);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	return &cache->entry;
+	return &resource->entry;
 }
 
 /**
@@ -10194,8 +10193,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 			 uint32_t group_id,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5_flow_tbl_resource *tbl;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 	struct mlx5_flow_cb_ctx ctx = {
@@ -10215,15 +10214,15 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 					  "cannot allocate ref memory");
 	}
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->handle->dvh.matcher = cache;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.matcher = resource;
 	return 0;
 }
 
@@ -10291,15 +10290,15 @@ flow_dv_tag_resource_register
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_tag_resource *cache_resource;
+	struct mlx5_flow_dv_tag_resource *resource;
 	struct mlx5_hlist_entry *entry;
 
 	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
 	if (entry) {
-		cache_resource = container_of
-			(entry, struct mlx5_flow_dv_tag_resource, entry);
-		dev_flow->handle->dvh.rix_tag = cache_resource->idx;
-		dev_flow->dv.tag_resource = cache_resource;
+		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
+					entry);
+		dev_flow->handle->dvh.rix_tag = resource->idx;
+		dev_flow->dv.tag_resource = resource;
 		return 0;
 	}
 	return -rte_errno;
@@ -10626,68 +10625,69 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused,
-			struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-
-	if (resource->ratio == cache_resource->ratio &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->ft_id == cache_resource->ft_id &&
-	    resource->set_action == cache_resource->set_action &&
-	    !memcmp((void *)&resource->sample_act,
-		    (void *)&cache_resource->sample_act,
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+
+	if (ctx_resource->ratio == resource->ratio &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    ctx_resource->ft_id == resource->ft_id &&
+	    ctx_resource->set_action == resource->set_action &&
+	    !memcmp((void *)&ctx_resource->sample_act,
+		    (void *)&resource->sample_act,
 		    sizeof(struct mlx5_flow_sub_actions_list))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
 		flow_dv_sample_sub_actions_release(dev,
-						&resource->sample_idx);
+						   &ctx_resource->sample_idx);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	void **sample_dv_actions = resource->sub_actions;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	void **sample_dv_actions = ctx_resource->sub_actions;
+	struct mlx5_flow_dv_sample_resource *resource;
 	struct mlx5dv_dr_flow_sampler_attr sampler_attr;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	struct mlx5_flow_tbl_resource *tbl;
 	uint32_t idx = 0;
 	const uint32_t next_ft_step = 1;
-	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
+	uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
 	uint8_t is_egress = 0;
 	uint8_t is_transfer = 0;
 	struct rte_flow_error *error = ctx->error;
 
 	/* Register new sample resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	/* Create normal path table level */
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		is_transfer = 1;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
 		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
 					is_egress, is_transfer,
@@ -10700,8 +10700,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 					  "for sample");
 		goto error;
 	}
-	cache_resource->normal_path_tbl = tbl;
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
+	resource->normal_path_tbl = tbl;
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
 		if (!sh->default_miss_action) {
 			rte_flow_error_set(error, ENOMEM,
 						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10710,33 +10710,33 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 						"created");
 			goto error;
 		}
-		sample_dv_actions[resource->sample_act.actions_num++] =
+		sample_dv_actions[ctx_resource->sample_act.actions_num++] =
 						sh->default_miss_action;
 	}
 	/* Create a DR sample action */
-	sampler_attr.sample_ratio = cache_resource->ratio;
+	sampler_attr.sample_ratio = resource->ratio;
 	sampler_attr.default_next_table = tbl->obj;
-	sampler_attr.num_sample_actions = resource->sample_act.actions_num;
+	sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
 	sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
 							&sample_dv_actions[0];
-	sampler_attr.action = cache_resource->set_action;
+	sampler_attr.action = resource->set_action;
 	if (mlx5_os_flow_dr_create_flow_action_sampler
-			(&sampler_attr, &cache_resource->verbs_action)) {
+			(&sampler_attr, &resource->verbs_action)) {
 		rte_flow_error_set(error, ENOMEM,
 					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					NULL, "cannot create sample action");
 		goto error;
 	}
-	cache_resource->idx = idx;
-	cache_resource->dev = dev;
-	return &cache_resource->entry;
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
 error:
-	if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
 		flow_dv_sample_sub_actions_release(dev,
-						   &cache_resource->sample_idx);
-	if (cache_resource->normal_path_tbl)
+						   &resource->sample_idx);
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-				cache_resource->normal_path_tbl);
+				resource->normal_path_tbl);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
 	return NULL;
 
@@ -10747,8 +10747,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to sample resource.
+ * @param[in] ref
+ *   Pointer to sample resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10759,66 +10759,66 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_sample_resource *resource,
+			 struct mlx5_flow_dv_sample_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_sample = cache_resource->idx;
-	dev_flow->dv.sample_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_sample = resource->idx;
+	dev_flow->dv.sample_res = resource;
 	return 0;
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused,
-			    struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
-	if (resource->num_of_dest == cache_resource->num_of_dest &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    !memcmp((void *)cache_resource->sample_act,
-		    (void *)resource->sample_act,
-		   (resource->num_of_dest *
+	if (ctx_resource->num_of_dest == resource->num_of_dest &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    !memcmp((void *)resource->sample_act,
+		    (void *)ctx_resource->sample_act,
+		   (ctx_resource->num_of_dest *
 		   sizeof(struct mlx5_flow_sub_actions_list)))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
-		for (idx = 0; idx < resource->num_of_dest; idx++)
+		for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 			flow_dv_sample_sub_actions_release(dev,
-					&resource->sample_idx[idx]);
+					&ctx_resource->sample_idx[idx]);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
 	struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -10831,23 +10831,23 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	int ret;
 
 	/* Register new destination array resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
 					    &res_idx);
-	if (!cache_resource) {
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
 				 mlx5_malloc(MLX5_MEM_ZERO,
 				 sizeof(struct mlx5dv_dr_action_dest_attr),
@@ -10860,7 +10860,7 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 			goto error;
 		}
 		dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
-		sample_act = &resource->sample_act[idx];
+		sample_act = &ctx_resource->sample_act[idx];
 		action_flags = sample_act->action_flags;
 		switch (action_flags) {
 		case MLX5_FLOW_ACTION_QUEUE:
@@ -10891,9 +10891,9 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	/* create a dest array actioin */
 	ret = mlx5_os_flow_dr_create_flow_action_dest_array
 						(domain,
-						 cache_resource->num_of_dest,
+						 resource->num_of_dest,
 						 dest_attr,
-						 &cache_resource->action);
+						 &resource->action);
 	if (ret) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10901,19 +10901,18 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 				   "cannot create destination array action");
 		goto error;
 	}
-	cache_resource->idx = res_idx;
-	cache_resource->dev = dev;
-	for (idx = 0; idx < resource->num_of_dest; idx++)
+	resource->idx = res_idx;
+	resource->dev = dev;
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 		mlx5_free(dest_attr[idx]);
-	return &cache_resource->entry;
+	return &resource->entry;
 error:
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[idx]);
+						   &resource->sample_idx[idx]);
 		if (dest_attr[idx])
 			mlx5_free(dest_attr[idx]);
 	}
-
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
 	return NULL;
 }
@@ -10923,8 +10922,8 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to destination array resource.
+ * @param[in] ref
+ *   Pointer to destination array resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10935,25 +10934,25 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_dest_array_resource *resource,
+			 struct mlx5_flow_dv_dest_array_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_dest_array = cache_resource->idx;
-	dev_flow->dv.dest_array_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_dest_array = resource->idx;
+	dev_flow->dv.dest_array_res = resource;
 	return 0;
 }
 
@@ -13382,14 +13381,15 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			  struct mlx5_cache_entry *entry)
+flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+			  struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache),
-							  entry);
+	struct mlx5_flow_dv_matcher *resource = container_of(entry,
+							     typeof(*resource),
+							     entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object));
-	mlx5_free(cache);
+	claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
+	mlx5_free(resource);
 }
 
 /**
@@ -13413,7 +13413,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13432,7 +13432,7 @@ flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
-		container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
@@ -13454,15 +13454,14 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
 				     uint32_t encap_decap_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-					encap_decap_idx);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+				  encap_decap_idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->action);
-	return mlx5_hlist_unregister(priv->sh->encaps_decaps,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
 }
 
 /**
@@ -13524,15 +13523,15 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry)
+flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_port_id_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+				  container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
 
 /**
@@ -13551,14 +13550,14 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 					uint32_t port_id)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->port_id_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13581,15 +13580,15 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry)
+flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
 
 /**
@@ -13608,15 +13607,15 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 					  struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx = handle->dvh.rix_push_vlan;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->push_vlan_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13653,26 +13652,24 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry)
+flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	if (cache_resource->verbs_action)
+	if (resource->verbs_action)
 		claim_zero(mlx5_flow_os_destroy_flow_action
-				(cache_resource->verbs_action));
-	if (cache_resource->normal_path_tbl)
+						      (resource->verbs_action));
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-			cache_resource->normal_path_tbl);
-	flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			cache_resource->idx);
-	DRV_LOG(DEBUG, "sample resource %p: removed",
-		(void *)cache_resource);
+					     resource->normal_path_tbl);
+	flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
+	DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
 }
 
 /**
@@ -13691,38 +13688,36 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 				     struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			 handle->dvh.rix_sample);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+				  handle->dvh.rix_sample);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->verbs_action);
-	return mlx5_cache_unregister(&priv->sh->sample_action_list,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->verbs_action);
+	return mlx5_list_unregister(&priv->sh->sample_action_list,
+				    &resource->entry);
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	uint32_t i = 0;
 
-	MLX5_ASSERT(cache_resource->action);
-	if (cache_resource->action)
-		claim_zero(mlx5_flow_os_destroy_flow_action
-					(cache_resource->action));
-	for (; i < cache_resource->num_of_dest; i++)
+	MLX5_ASSERT(resource->action);
+	if (resource->action)
+		claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	for (; i < resource->num_of_dest; i++)
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[i]);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			cache_resource->idx);
+						   &resource->sample_idx[i]);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
 	DRV_LOG(DEBUG, "destination array resource %p: removed",
-		(void *)cache_resource);
+		(void *)resource);
 }
 
 /**
@@ -13741,15 +13736,15 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 				    struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_dest_array_resource *cache;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			       handle->dvh.rix_dest_array);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				  handle->dvh.rix_dest_array);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->dest_array_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->dest_array_list,
+				    &resource->entry);
 }
 
 static void
@@ -14592,7 +14587,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 		if (sub_policy->color_matcher[i]) {
 			tbl = container_of(sub_policy->color_matcher[i]->tbl,
 				typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 				      &sub_policy->color_matcher[i]->entry);
 			sub_policy->color_matcher[i] = NULL;
 		}
@@ -15326,8 +15321,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_cache_unregister(&tbl->matchers,
-				      &mtrmng->def_matcher[i]->entry);
+			mlx5_list_unregister(&tbl->matchers,
+					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
 		for (j = 0; j < MLX5_REG_BITS; j++) {
@@ -15336,8 +15331,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_cache_unregister(&tbl->matchers,
-					&mtrmng->drop_matcher[i][j]->entry);
+				mlx5_list_unregister(&tbl->matchers,
+					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
 		}
@@ -15433,7 +15428,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 			bool match_src_port,
 			struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
@@ -15469,7 +15464,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -15835,7 +15830,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 						     0, &error);
 	uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
 	uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
 			.size = sizeof(matcher.mask.buf) -
@@ -15881,7 +15876,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -15918,7 +15913,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
@@ -16104,7 +16099,6 @@ flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
 	return NULL;
 }
 
-
 /**
  * Destroy the sub policy table with RX queue.
  *
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 1b264e5994..3dcc71d51d 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index bb9a908087..8395332507 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index f9557c09ff..4536ca807d 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,29 +9,29 @@
 #include "mlx5_utils.h"
 
 
-/********************* Cache list ************************/
+/********************* MLX5 list ************************/
 
-static struct mlx5_cache_entry *
-mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry __rte_unused,
+static struct mlx5_list_entry *
+mlx5_list_default_create_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry __rte_unused,
 			     void *ctx __rte_unused)
 {
 	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
 }
 
 static void
-mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
 }
 
 int
-mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
+mlx5_list_create(struct mlx5_list *list, const char *name,
 		     uint32_t entry_size, void *ctx,
-		     mlx5_cache_create_cb cb_create,
-		     mlx5_cache_match_cb cb_match,
-		     mlx5_cache_remove_cb cb_remove)
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove)
 {
 	MLX5_ASSERT(list);
 	if (!cb_match || (!cb_create ^ !cb_remove))
@@ -40,19 +40,19 @@ mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
+	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
+	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
 	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	LIST_INIT(&list->head);
 	return 0;
 }
 
-static struct mlx5_cache_entry *
-__cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	LIST_FOREACH(entry, &list->head, next) {
 		if (list->cb_match(list, entry, ctx))
@@ -60,7 +60,7 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 		if (reuse) {
 			__atomic_add_fetch(&entry->ref_cnt, 1,
 					   __ATOMIC_RELAXED);
-			DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
 		break;
@@ -68,33 +68,33 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_cache_entry *
-cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __cache_lookup(list, ctx, reuse);
+	entry = __list_lookup(list, ctx, reuse);
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	return cache_lookup(list, ctx, false);
+	return list_lookup(list, ctx, false);
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	uint32_t prev_gen_cnt = 0;
 
 	MLX5_ASSERT(list);
 	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
 	/* Lookup with read lock, reuse if found. */
-	entry = cache_lookup(list, ctx, true);
+	entry = list_lookup(list, ctx, true);
 	if (entry)
 		return entry;
 	/* Not found, append with write lock - block read from other threads. */
@@ -102,13 +102,13 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __cache_lookup(list, ctx, true);
+		entry = __list_lookup(list, ctx, true);
 		if (entry)
 			goto done;
 	}
 	entry = list->cb_create(list, entry, ctx);
 	if (!entry) {
-		DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
+		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
 			list->name, (void *)entry);
 		goto done;
 	}
@@ -116,7 +116,7 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	LIST_INSERT_HEAD(&list->head, entry, next);
 	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 done:
 	rte_rwlock_write_unlock(&list->lock);
@@ -124,12 +124,12 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 }
 
 int
-mlx5_cache_unregister(struct mlx5_cache_list *list,
-		      struct mlx5_cache_entry *entry)
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
 {
 	rte_rwlock_write_lock(&list->lock);
 	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 	if (--entry->ref_cnt) {
 		rte_rwlock_write_unlock(&list->lock);
@@ -140,15 +140,15 @@ mlx5_cache_unregister(struct mlx5_cache_list *list,
 	LIST_REMOVE(entry, next);
 	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 		list->name, (void *)entry);
 	return 0;
 }
 
 void
-mlx5_cache_list_destroy(struct mlx5_cache_list *list)
+mlx5_list_destroy(struct mlx5_list *list)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	MLX5_ASSERT(list);
 	/* no LIST_FOREACH_SAFE, using while instead */
@@ -156,14 +156,14 @@ mlx5_cache_list_destroy(struct mlx5_cache_list *list)
 		entry = LIST_FIRST(&list->head);
 		LIST_REMOVE(entry, next);
 		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
 			list->name, (void *)entry);
 	}
 	memset(list, 0, sizeof(*list));
 }
 
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
+mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
 	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index a509b0a4eb..cfb3cb6180 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -297,19 +297,19 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ cache list *****************************/
+/************************ mlx5 list *****************************/
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
 
-struct mlx5_cache_list;
+struct mlx5_list;
 
 /**
- * Structure of the entry in the cache list, user should define its own struct
+ * Structure of the entry in the mlx5 list, user should define its own struct
  * that contains this in order to store the data.
  */
-struct mlx5_cache_entry {
-	LIST_ENTRY(mlx5_cache_entry) next; /* Entry pointers in the list. */
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* Reference count. */
 };
 
@@ -317,18 +317,18 @@ struct mlx5_cache_entry {
  * Type of callback function for entry removal.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
-				     struct mlx5_cache_entry *entry);
+typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
+				     struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -337,14 +337,14 @@ typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
-				   struct mlx5_cache_entry *entry, void *ctx);
+typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+				   struct mlx5_list_entry *entry, void *ctx);
 
 /**
- * Type of function for user defined cache list entry creation.
+ * Type of function for user defined mlx5 list entry creation.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The new allocated entry, NULL if list entry size unspecified,
  *   New entry has to be allocated in callback and return.
@@ -354,46 +354,46 @@ typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_cache_entry *(*mlx5_cache_create_cb)
-				 (struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry,
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry,
 				  void *ctx);
 
 /**
- * Linked cache list structure.
+ * Linked mlx5 list structure.
  *
- * Entry in cache list could be reused if entry already exists,
+ * Entry in mlx5 list could be reused if entry already exists,
  * reference count will increase and the existing entry returns.
  *
  * When destroy an entry from list, decrease reference count and only
  * destroy when no further reference.
  *
- * Linked list cache is designed for limited number of entries cache,
+ * Linked list is designed for limited number of entries,
  * read mostly, less modification.
  *
- * For huge amount of entries cache, please consider hash list cache.
+ * For huge amount of entries, please consider hash list.
  *
  */
-struct mlx5_cache_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the cache list. */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	mlx5_cache_create_cb cb_create; /**< entry create callback. */
-	mlx5_cache_match_cb cb_match; /**< entry match callback. */
-	mlx5_cache_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_cache_head, mlx5_cache_entry) head;
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
 };
 
 /**
- * Initialize a cache list.
+ * Create a mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
  * @param name
- *   Name of the cache list.
+ *   Name of the mlx5 list.
  * @param entry_size
  *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
@@ -407,11 +407,11 @@ struct mlx5_cache_list {
  * @return
  *   0 on success, otherwise failure.
  */
-int mlx5_cache_list_init(struct mlx5_cache_list *list,
+int mlx5_list_create(struct mlx5_list *list,
 			 const char *name, uint32_t entry_size, void *ctx,
-			 mlx5_cache_create_cb cb_create,
-			 mlx5_cache_match_cb cb_match,
-			 mlx5_cache_remove_cb cb_remove);
+			 mlx5_list_create_cb cb_create,
+			 mlx5_list_match_cb cb_match,
+			 mlx5_list_remove_cb cb_remove);
 
 /**
  * Search an entry matching the key.
@@ -420,18 +420,18 @@ int mlx5_cache_list_init(struct mlx5_cache_list *list,
  * this function only in main thread.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  * @param ctx
  *   Common context parameter used by entry callback function.
  *
  * @return
- *   Pointer of the cache entry if found, NULL otherwise.
+ *   Pointer of the list entry if found, NULL otherwise.
  */
-struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
 					   void *ctx);
 
 /**
- * Reuse or create an entry to the cache list.
+ * Reuse or create an entry to the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
@@ -441,42 +441,42 @@ struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
  * @return
  *   registered entry on success, NULL otherwise
  */
-struct mlx5_cache_entry *mlx5_cache_register(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
 					     void *ctx);
 
 /**
- * Remove an entry from the cache list.
+ * Remove an entry from the mlx5 list.
  *
  * User should guarantee the validity of the entry.
  *
  * @param list
  *   Pointer to the hast list.
  * @param entry
- *   Entry to be removed from the cache list table.
+ *   Entry to be removed from the mlx5 list table.
  * @return
  *   0 on entry removed, 1 on entry still referenced.
  */
-int mlx5_cache_unregister(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry);
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
 
 /**
- * Destroy the cache list.
+ * Destroy the mlx5 list.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  */
-void mlx5_cache_list_destroy(struct mlx5_cache_list *list);
+void mlx5_list_destroy(struct mlx5_list *list);
 
 /**
- * Get entry number from the cache list.
+ * Get entry number from the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list.
  * @return
- *   Cache list entry number.
+ *   mlx5 list entry number.
  */
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list);
+mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /********************************* indexed pool *************************/
 
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 17716b66c9..bcf72dc6db 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,10 +610,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v4 08/26] net/mlx5: add per lcore cache to the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When mlx5 list object is accessed by multiple cores, the list lock
counter is all the time written by all the cores what increases cache
misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup
operation, all the other threads coming to do an operation in the list
are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when
the list objects are mostly reused.

Synchronization with atomic operations should be done in order to
allow threads to unregister an entry from other thread cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 57b0a1c57f..d7119dd561 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1704,10 +1712,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index bf1fbb530b..740695ea5c 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1365,6 +1365,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 4dec703366..ce363355c1 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1634,7 +1634,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1642,6 +1646,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1650,6 +1659,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1658,6 +1672,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index d19b41c20a..10aa557fee 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3820,6 +3820,39 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_port_id_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate port_id action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
+}
+
 /**
  * Find existing table port ID resource or create and register a new one.
  *
@@ -3912,6 +3945,39 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate push_vlan action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
+}
+
 /**
  * Find existing push vlan resource or create and register a new one.
  *
@@ -9885,6 +9951,36 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 	return match_criteria_enable;
 }
 
+static struct mlx5_list_entry *
+flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_matcher *ref = ctx->data;
+	struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
+							    typeof(*tbl), tbl);
+	struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
+							    sizeof(*resource),
+							    0, SOCKET_ID_ANY);
+
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot create matcher");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->tbl = &tbl->tbl;
+	return &resource->entry;
+}
+
+static void
+flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 struct mlx5_hlist_entry *
 flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 {
@@ -9951,10 +10047,12 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
 			 flow_dv_matcher_create_cb,
 			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb);
+			 flow_dv_matcher_remove_cb,
+			 flow_dv_matcher_clone_cb,
+			 flow_dv_matcher_clone_free_cb);
 	return &tbl_data->entry;
 }
 
@@ -10742,6 +10840,45 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 
 }
 
+struct mlx5_list_entry *
+flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t idx = 0;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_sample_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+			resource->idx);
+}
+
 /**
  * Find existing sample resource or create and register a new one.
  *
@@ -10917,6 +11054,46 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 	return NULL;
 }
 
+struct mlx5_list_entry *
+flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t res_idx = 0;
+	struct rte_flow_error *error = ctx->error;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				      &res_idx);
+	if (!resource) {
+		rte_flow_error_set(error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate dest-array memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = res_idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
+}
+
 /**
  * Find existing destination array resource or create and register a new one.
  *
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 3dcc71d51d..5450ddd388 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 8395332507..f8769da8dc 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1857,20 +1857,18 @@ mlx5_ind_table_obj_get(struct rte_eth_dev *dev, const uint16_t *queues,
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_ind_table_obj *ind_tbl;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		if ((ind_tbl->queues_n == queues_n) &&
 		    (memcmp(ind_tbl->queues, queues,
 			    ind_tbl->queues_n * sizeof(ind_tbl->queues[0]))
-		     == 0))
+		     == 0)) {
+			__atomic_fetch_add(&ind_tbl->refcnt, 1,
+					   __ATOMIC_RELAXED);
 			break;
+		}
 	}
-	if (ind_tbl) {
-		unsigned int i;
-
-		__atomic_fetch_add(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
-		for (i = 0; i != ind_tbl->queues_n; ++i)
-			mlx5_rxq_get(dev, ind_tbl->queues[i]);
-	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ind_tbl;
 }
 
@@ -1893,19 +1891,20 @@ mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
 			   bool standalone)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	unsigned int i;
+	unsigned int i, ret;
 
-	if (__atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED) == 0)
-		priv->obj_ops.ind_table_destroy(ind_tbl);
+	rte_rwlock_write_lock(&priv->ind_tbls_lock);
+	ret = __atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
+	if (!ret && !standalone)
+		LIST_REMOVE(ind_tbl, next);
+	rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	if (ret)
+		return 1;
+	priv->obj_ops.ind_table_destroy(ind_tbl);
 	for (i = 0; i != ind_tbl->queues_n; ++i)
 		claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
-	if (__atomic_load_n(&ind_tbl->refcnt, __ATOMIC_RELAXED) == 0) {
-		if (!standalone)
-			LIST_REMOVE(ind_tbl, next);
-		mlx5_free(ind_tbl);
-		return 0;
-	}
-	return 1;
+	mlx5_free(ind_tbl);
+	return 0;
 }
 
 /**
@@ -1924,12 +1923,14 @@ mlx5_ind_table_obj_verify(struct rte_eth_dev *dev)
 	struct mlx5_ind_table_obj *ind_tbl;
 	int ret = 0;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		DRV_LOG(DEBUG,
 			"port %u indirection table obj %p still referenced",
 			dev->data->port_id, (void *)ind_tbl);
 		++ret;
 	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ret;
 }
 
@@ -2015,8 +2016,11 @@ mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues,
 		mlx5_free(ind_tbl);
 		return NULL;
 	}
-	if (!standalone)
+	if (!standalone) {
+		rte_rwlock_write_lock(&priv->ind_tbls_lock);
 		LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
+		rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	}
 	return ind_tbl;
 }
 
@@ -2328,6 +2332,35 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 	return hrxq ? &hrxq->entry : NULL;
 }
 
+struct mlx5_list_entry *
+mlx5_hrxq_clone_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry,
+		    void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq;
+	uint32_t hrxq_idx = 0;
+
+	hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx);
+	if (!hrxq)
+		return NULL;
+	memcpy(hrxq, entry, sizeof(*hrxq) + MLX5_RSS_HASH_KEY_LEN);
+	hrxq->idx = hrxq_idx;
+	return &hrxq->entry;
+}
+
+void
+mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq->idx);
+}
+
 /**
  * Get an Rx Hash queue.
  *
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 4536ca807d..f505caed4e 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,57 +9,68 @@
 #include "mlx5_utils.h"
 
 
-/********************* MLX5 list ************************/
-
-static struct mlx5_list_entry *
-mlx5_list_default_create_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
-{
-	mlx5_free(entry);
-}
+/********************* mlx5 list ************************/
 
 int
-mlx5_list_create(struct mlx5_list *list, const char *name,
-		     uint32_t entry_size, void *ctx,
-		     mlx5_list_create_cb cb_create,
-		     mlx5_list_match_cb cb_match,
-		     mlx5_list_remove_cb cb_remove)
+mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	int i;
+
 	MLX5_ASSERT(list);
-	if (!cb_match || (!cb_create ^ !cb_remove))
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free)
 		return -1;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
+	list->cb_create = cb_create;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	LIST_INIT(&list->head);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry;
-
-	LIST_FOREACH(entry, &list->head, next) {
-		if (list->cb_match(list, entry, ctx))
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (list->cb_match(list, entry, ctx)) {
+			if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_ACQUIRE);
+				if (ret == 0) {
+					LIST_REMOVE(entry, next);
+					list->cb_clone_free(list, entry);
+				}
+			}
+			entry = nentry;
 			continue;
+		}
 		if (reuse) {
-			__atomic_add_fetch(&entry->ref_cnt, 1,
-					   __ATOMIC_RELAXED);
+			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+						 __ATOMIC_ACQUIRE);
+			if (ret == 1u) {
+				/* Entry was invalid before, free it. */
+				LIST_REMOVE(entry, next);
+				list->cb_clone_free(list, entry);
+				entry = nentry;
+				continue;
+			}
 			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
@@ -68,96 +79,141 @@ __list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_list_entry *
-list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry = NULL;
+	int i;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, ctx, reuse);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	return list_lookup(list, ctx, false);
+	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+
+	if (!lentry)
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
 }
 
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry, *lentry;
 	uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
-	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* Lookup in local cache. */
+	lentry = __list_lookup(list, lcore_index, ctx, true);
+	if (lentry)
+		return lentry;
 	/* Lookup with read lock, reuse if found. */
-	entry = list_lookup(list, ctx, true);
-	if (entry)
-		return entry;
+	rte_rwlock_read_lock(&list->lock);
+	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+	if (entry == NULL) {
+		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
+					       __ATOMIC_ACQUIRE);
+		rte_rwlock_read_unlock(&list->lock);
+	} else {
+		rte_rwlock_read_unlock(&list->lock);
+		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+	}
 	/* Not found, append with write lock - block read from other threads. */
 	rte_rwlock_write_lock(&list->lock);
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, ctx, true);
-		if (entry)
-			goto done;
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (entry) {
+			rte_rwlock_write_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
 	}
 	entry = list->cb_create(list, entry, ctx);
-	if (!entry) {
-		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
-			list->name, (void *)entry);
-		goto done;
+	if (entry) {
+		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+		if (!lentry) {
+			list->cb_remove(list, entry);
+		} else {
+			entry->ref_cnt = 1u;
+			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
+					 next);
+			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
+			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+				list->name, (void *)entry, entry->ref_cnt);
+		}
+
 	}
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(&list->head, entry, next);
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-done:
 	rte_rwlock_write_unlock(&list->lock);
-	return entry;
+	return lentry;
 }
 
 int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
+	struct mlx5_list_entry *gentry = entry->gentry;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
+	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
+		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		LIST_REMOVE(gentry, next);
+		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
-		return 1;
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
 	}
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-	__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	LIST_REMOVE(entry, next);
-	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-		list->name, (void *)entry);
-	return 0;
+	return 1;
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
+	int i;
 
 	MLX5_ASSERT(list);
-	/* no LIST_FOREACH_SAFE, using while instead */
-	while (!LIST_EMPTY(&list->head)) {
-		entry = LIST_FIRST(&list->head);
-		LIST_REMOVE(entry, next);
-		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
-			list->name, (void *)entry);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list, entry);
+			}
+		}
 	}
 	memset(list, 0, sizeof(*list));
 }
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index cfb3cb6180..9e3fe0cb85 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -310,9 +310,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -340,6 +345,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -376,15 +388,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -394,8 +408,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -408,10 +420,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index bcf72dc6db..8ced98f0dc 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,9 +610,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v4 09/26] net/mlx5: minimize list critical sections

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is
taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical
sections.

These operations are heavy and make the critical sections heavy because
they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation
counter in order to detect parallel allocations.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index f505caed4e..c4c9adb039 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 9e3fe0cb85..6dade8238d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -388,8 +388,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
-- 
2.25.1

[dpdk-dev] [PATCH v4 10/26] net/mlx5: manage list cache entries release

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When a cache entry is allocated by lcore A and is released by lcore B,
the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be
increased atomically when the non-original lcore decreases the reference
counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in
its cache, it will check the counter in order to free invalid entries in
its cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index c4c9adb039..13c7dbe1c2 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 6dade8238d..71da5ab4f9 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -311,11 +311,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v4 11/26] net/mlx5: relax the list utility atomic operations

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The atomic operation in the list utility no need a barriers because the
critical part are managed by RW lock.

Relax them.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 13c7dbe1c2..daecf37575 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
-- 
2.25.1

[dpdk-dev] [PATCH v4 12/26] net/mlx5: allocate list memory by the create API

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence
with the hlist utility.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  56 ++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 137 insertions(+), 81 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index d7119dd561..da4d2fdadc 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -447,6 +455,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -508,9 +532,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1712,11 +1750,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1775,7 +1815,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f51be5ace6..6661d041ed 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1609,7 +1609,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 740695ea5c..5da3d93a5b 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ce363355c1..ce4d205e86 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 10aa557fee..982056cb41 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3883,7 +3883,7 @@ flow_dv_port_id_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -4008,7 +4008,7 @@ flow_dv_push_vlan_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10047,12 +10047,22 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
-			 flow_dv_matcher_create_cb,
-			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb,
-			 flow_dv_matcher_clone_cb,
-			 flow_dv_matcher_clone_free_cb);
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+					      flow_dv_matcher_create_cb,
+					      flow_dv_matcher_match_cb,
+					      flow_dv_matcher_remove_cb,
+					      flow_dv_matcher_clone_cb,
+					      flow_dv_matcher_clone_free_cb);
+	if (!tbl_data->matchers) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot create tbl matcher list");
+		mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
+		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
+		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
+		return NULL;
+	}
 	return &tbl_data->entry;
 }
 
@@ -10180,7 +10190,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10312,7 +10322,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
@@ -10909,7 +10919,7 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -11124,7 +11134,7 @@ flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -13590,7 +13600,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13733,7 +13743,7 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+	return mlx5_list_unregister(priv->sh->port_id_action_list,
 				    &resource->entry);
 }
 
@@ -13791,7 +13801,7 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+	return mlx5_list_unregister(priv->sh->push_vlan_action_list,
 				    &resource->entry);
 }
 
@@ -13872,7 +13882,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->verbs_action);
-	return mlx5_list_unregister(&priv->sh->sample_action_list,
+	return mlx5_list_unregister(priv->sh->sample_action_list,
 				    &resource->entry);
 }
 
@@ -13920,7 +13930,7 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->dest_array_list,
+	return mlx5_list_unregister(priv->sh->dest_array_list,
 				    &resource->entry);
 }
 
@@ -14764,7 +14774,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 		if (sub_policy->color_matcher[i]) {
 			tbl = container_of(sub_policy->color_matcher[i]->tbl,
 				typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 				      &sub_policy->color_matcher[i]->entry);
 			sub_policy->color_matcher[i] = NULL;
 		}
@@ -15498,7 +15508,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
@@ -15508,7 +15518,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_list_unregister(&tbl->matchers,
+				mlx5_list_unregister(tbl->matchers,
 					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
@@ -15641,7 +15651,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -16053,7 +16063,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -16090,7 +16100,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index f8769da8dc..aa9e973d10 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index daecf37575..0be778935f 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 71da5ab4f9..ea64bb75c9 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -420,15 +420,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 8ced98f0dc..e6176e70d2 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v4 13/26] common/mlx5: move list utility to common

[Suanming Mou]

Hash list is planned to be implemented with the cache list code.

This commit moves the list utility to common directory.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 250 +++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h | 205 +++++++++++++++++++
 drivers/common/mlx5/version.map         |   7 +
 drivers/net/mlx5/mlx5_utils.c           | 251 ------------------------
 drivers/net/mlx5/mlx5_utils.h           | 197 -------------------
 6 files changed, 464 insertions(+), 448 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 306f2f1ab7..7fb7d40b38 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index ad2011e858..8bb8a6016d 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -11,6 +11,256 @@
 #include "mlx5_common_utils.h"
 #include "mlx5_common_log.h"
 
+/********************* mlx5 list ************************/
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+	int i;
+
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (name)
+		snprintf(list->name, sizeof(list->name), "%s", name);
+	list->ctx = ctx;
+	list->cb_create = cb_create;
+	list->cb_match = cb_match;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
+	rte_rwlock_init(&list->lock);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
+	return list;
+}
+
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+{
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_RELAXED) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_RELAXED);
+			}
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
+		}
+		entry = LIST_NEXT(entry, next);
+	}
+	return NULL;
+}
+
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry = NULL;
+	int i;
+
+	rte_rwlock_read_lock(&list->lock);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
+	rte_rwlock_read_unlock(&list->lock);
+	return entry;
+}
+
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
+{
+	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+
+	if (unlikely(!lentry))
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
+}
+
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	MLX5_ASSERT(list);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
+	rte_rwlock_read_lock(&list->lock);
+	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+	if (likely(entry)) {
+		rte_rwlock_read_unlock(&list->lock);
+		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+	}
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
+	rte_rwlock_write_lock(&list->lock);
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
+			rte_rwlock_write_unlock(&list->lock);
+			list->cb_remove(list, entry);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
+		}
+	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
+	rte_rwlock_write_unlock(&list->lock);
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
+	return local_entry;
+}
+
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	rte_rwlock_write_lock(&list->lock);
+	if (likely(gentry->ref_cnt == 0)) {
+		LIST_REMOVE(gentry, next);
+		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
+	}
+	rte_rwlock_write_unlock(&list->lock);
+	return 1;
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	struct mlx5_list_entry *entry;
+	int i;
+
+	MLX5_ASSERT(list);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list, entry);
+			}
+		}
+	}
+	mlx5_free(list);
+}
+
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list)
+{
+	MLX5_ASSERT(list);
+	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+}
+
 /********************* Hash List **********************/
 
 static struct mlx5_hlist_entry *
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index ed378ce9bd..96add6d003 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -7,6 +7,211 @@
 
 #include "mlx5_common.h"
 
+/************************ mlx5 list *****************************/
+
+/** Maximum size of string for naming. */
+#define MLX5_NAME_SIZE			32
+
+struct mlx5_list;
+
+/**
+ * Structure of the entry in the mlx5 list, user should define its own struct
+ * that contains this in order to store the data.
+ */
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
+	struct mlx5_list_entry *gentry;
+};
+
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
+} __rte_cache_aligned;
+
+/**
+ * Type of callback function for entry removal.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The entry in the list.
+ */
+typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
+				     struct mlx5_list_entry *entry);
+
+/**
+ * Type of function for user defined matching.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The entry in the list.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   0 if matching, non-zero number otherwise.
+ */
+typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+				   struct mlx5_list_entry *entry, void *ctx);
+
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
+/**
+ * Type of function for user defined mlx5 list entry creation.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The new allocated entry, NULL if list entry size unspecified,
+ *   New entry has to be allocated in callback and return.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   Pointer of entry on success, NULL otherwise.
+ */
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry,
+				  void *ctx);
+
+/**
+ * Linked mlx5 list structure.
+ *
+ * Entry in mlx5 list could be reused if entry already exists,
+ * reference count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ *
+ * Linked list is designed for limited number of entries,
+ * read mostly, less modification.
+ *
+ * For huge amount of entries, please consider hash list.
+ *
+ */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
+	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+};
+
+/**
+ * Create a mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param name
+ *   Name of the mlx5 list.
+ * @param ctx
+ *   Pointer to the list context data.
+ * @param cb_create
+ *   Callback function for entry create.
+ * @param cb_match
+ *   Callback function for entry match.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @return
+ *   List pointer on success, otherwise NULL.
+ */
+__rte_internal
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ * @param ctx
+ *   Common context parameter used by entry callback function.
+ *
+ * @return
+ *   Pointer of the list entry if found, NULL otherwise.
+ */
+__rte_internal
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
+					   void *ctx);
+
+/**
+ * Reuse or create an entry to the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param ctx
+ *   Common context parameter used by callback function.
+ *
+ * @return
+ *   registered entry on success, NULL otherwise
+ */
+__rte_internal
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
+					     void *ctx);
+
+/**
+ * Remove an entry from the mlx5 list.
+ *
+ * User should guarantee the validity of the entry.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @param entry
+ *   Entry to be removed from the mlx5 list table.
+ * @return
+ *   0 on entry removed, 1 on entry still referenced.
+ */
+__rte_internal
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
+
+/**
+ * Destroy the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ */
+__rte_internal
+void mlx5_list_destroy(struct mlx5_list *list);
+
+/**
+ * Get entry number from the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @return
+ *   mlx5 list entry number.
+ */
+__rte_internal
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list);
+
+/************************ Hash list *****************************/
+
 #define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
 #define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
 
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index b8be73a77b..e6586d6f6f 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -73,6 +73,13 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_list_create;
+	mlx5_list_register;
+	mlx5_list_unregister;
+	mlx5_list_lookup;
+	mlx5_list_get_entry_num;
+	mlx5_list_destroy;
+
 	mlx5_hlist_create;
 	mlx5_hlist_lookup;
 	mlx5_hlist_register;
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 0be778935f..e4e66ae4c5 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -8,257 +8,6 @@
 
 #include "mlx5_utils.h"
 
-
-/********************* mlx5 list ************************/
-
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
-{
-	struct mlx5_list *list;
-	int i;
-
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return NULL;
-	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
-	return list;
-}
-
-static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
-{
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
-	uint32_t ret;
-
-	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
-			if (reuse) {
-				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_RELAXED) - 1;
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
-					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
-				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_RELAXED);
-			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
-				return entry;
-			if (reuse && ret == 0)
-				entry->ref_cnt--; /* Invalid entry. */
-		}
-		entry = LIST_NEXT(entry, next);
-	}
-	return NULL;
-}
-
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry = NULL;
-	int i;
-
-	rte_rwlock_read_lock(&list->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
-		if (entry)
-			break;
-	}
-	rte_rwlock_read_unlock(&list->lock);
-	return entry;
-}
-
-static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
-		       struct mlx5_list_entry *gentry, void *ctx)
-{
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
-
-	if (unlikely(!lentry))
-		return NULL;
-	lentry->ref_cnt = 1u;
-	lentry->gentry = gentry;
-	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
-	return lentry;
-}
-
-static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
-{
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
-	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
-	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
-					       __ATOMIC_RELAXED);
-
-	while (inv_cnt != 0 && entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
-			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
-			inv_cnt--;
-		}
-		entry = nentry;
-	}
-}
-
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry, *local_entry;
-	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
-	MLX5_ASSERT(list);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
-	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
-	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
-	if (local_entry)
-		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
-		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
-	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
-	if (unlikely(!entry))
-		return NULL;
-	local_entry = list->cb_clone(list, entry, ctx);
-	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
-		return NULL;
-	}
-	entry->ref_cnt = 1u;
-	local_entry->ref_cnt = 1u;
-	local_entry->gentry = entry;
-	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
-	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
-							       RTE_MAX_LCORE,
-							       ctx, true);
-
-		if (unlikely(oentry)) {
-			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
-		}
-	}
-	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
-		(void *)entry, entry->ref_cnt);
-	return local_entry;
-}
-
-int
-mlx5_list_unregister(struct mlx5_list *list,
-		      struct mlx5_list_entry *entry)
-{
-	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
-
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
-	if (entry->lcore_idx == (uint32_t)lcore_idx) {
-		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
-	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
-				   __ATOMIC_RELAXED);
-	} else {
-		return 0;
-	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	rte_rwlock_write_lock(&list->lock);
-	if (likely(gentry->ref_cnt == 0)) {
-		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
-		return 0;
-	}
-	rte_rwlock_write_unlock(&list->lock);
-	return 1;
-}
-
-void
-mlx5_list_destroy(struct mlx5_list *list)
-{
-	struct mlx5_list_entry *entry;
-	int i;
-
-	MLX5_ASSERT(list);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
-			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
-					(void *)entry);
-			} else {
-				list->cb_clone_free(list, entry);
-			}
-		}
-	}
-	mlx5_free(list);
-}
-
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list)
-{
-	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
-}
-
 /********************* Indexed pool **********************/
 
 static inline void
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index ea64bb75c9..cf3db89403 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -297,203 +297,6 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ mlx5 list *****************************/
-
-/** Maximum size of string for naming. */
-#define MLX5_NAME_SIZE			32
-
-struct mlx5_list;
-
-/**
- * Structure of the entry in the mlx5 list, user should define its own struct
- * that contains this in order to store the data.
- */
-struct mlx5_list_entry {
-	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* 0 means, entry is invalid. */
-	uint32_t lcore_idx;
-	struct mlx5_list_entry *gentry;
-};
-
-struct mlx5_list_cache {
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
-	uint32_t inv_cnt; /* Invalid entries counter. */
-} __rte_cache_aligned;
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
-				   struct mlx5_list_entry *entry, void *ctx);
-
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
-
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined mlx5 list entry creation.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer of entry on success, NULL otherwise.
- */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
-
-/**
- * Linked mlx5 list structure.
- *
- * Entry in mlx5 list could be reused if entry already exists,
- * reference count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- *
- * Linked list is designed for limited number of entries,
- * read mostly, less modification.
- *
- * For huge amount of entries, please consider hash list.
- *
- */
-struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
-	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-};
-
-/**
- * Create a mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param name
- *   Name of the mlx5 list.
- * @param ctx
- *   Pointer to the list context data.
- * @param cb_create
- *   Callback function for entry create.
- * @param cb_match
- *   Callback function for entry match.
- * @param cb_remove
- *   Callback function for entry remove.
- * @return
- *   List pointer on success, otherwise NULL.
- */
-struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
-				   mlx5_list_create_cb cb_create,
-				   mlx5_list_match_cb cb_match,
-				   mlx5_list_remove_cb cb_remove,
-				   mlx5_list_clone_cb cb_clone,
-				   mlx5_list_clone_free_cb cb_clone_free);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param list
- *   Pointer to the mlx5 list.
- * @param ctx
- *   Common context parameter used by entry callback function.
- *
- * @return
- *   Pointer of the list entry if found, NULL otherwise.
- */
-struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
-					   void *ctx);
-
-/**
- * Reuse or create an entry to the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param ctx
- *   Common context parameter used by callback function.
- *
- * @return
- *   registered entry on success, NULL otherwise
- */
-struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
-					     void *ctx);
-
-/**
- * Remove an entry from the mlx5 list.
- *
- * User should guarantee the validity of the entry.
- *
- * @param list
- *   Pointer to the hast list.
- * @param entry
- *   Entry to be removed from the mlx5 list table.
- * @return
- *   0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_list_unregister(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry);
-
-/**
- * Destroy the mlx5 list.
- *
- * @param list
- *   Pointer to the mlx5 list.
- */
-void mlx5_list_destroy(struct mlx5_list *list);
-
-/**
- * Get entry number from the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list.
- * @return
- *   mlx5 list entry number.
- */
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list);
-
 /********************************* indexed pool *************************/
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v4 14/26] common/mlx5: add list lcore share

[Suanming Mou]

As some actions in SW-steering is only memory and can be allowed to
create duplicate objects, for lists which no need to check if there
are existing same objects in other sub local lists, search the object
only in local list will be more efficient.

This commit adds the lcore share mode to list optimized the list
register.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 46 +++++++++++++++++++------
 drivers/common/mlx5/mlx5_common_utils.h | 16 ++++++---
 drivers/net/mlx5/linux/mlx5_os.c        | 11 +++---
 drivers/net/mlx5/mlx5_flow_dv.c         |  2 +-
 drivers/net/mlx5/windows/mlx5_os.c      |  2 +-
 5 files changed, 55 insertions(+), 22 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 8bb8a6016d..6ac78ba97f 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -14,7 +14,7 @@
 /********************* mlx5 list ************************/
 
 struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
@@ -35,6 +35,7 @@ mlx5_list_create(const char *name, void *ctx,
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
+	list->lcores_share = lcores_share;
 	list->cb_create = cb_create;
 	list->cb_match = cb_match;
 	list->cb_remove = cb_remove;
@@ -119,7 +120,10 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
+			if (list->lcores_share)
+				list->cb_clone_free(list, entry);
+			else
+				list->cb_remove(list, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -145,25 +149,36 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
 		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
 	/* 3. Prepare new entry for global list and for cache. */
 	entry = list->cb_create(list, entry, ctx);
 	if (unlikely(!entry))
 		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+		return entry;
+	}
 	local_entry = list->cb_clone(list, entry, ctx);
 	if (unlikely(!local_entry)) {
 		list->cb_remove(list, entry);
 		return NULL;
 	}
-	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
@@ -207,13 +222,22 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
+		if (list->lcores_share)
+			list->cb_clone_free(list, entry);
+		else
+			list->cb_remove(list, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 96add6d003..000279d236 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -100,11 +100,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -112,17 +109,27 @@ struct mlx5_list {
 	mlx5_list_clone_free_cb cb_clone_free;
 	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
 };
 
 /**
  * Create a mlx5 list.
  *
+ * For actions in SW-steering is only memory and  can be allowed
+ * to create duplicate objects, the lists don't need to check if
+ * there are existing same objects in other sub local lists,
+ * search the object only in local list will be more efficient.
+ *
  * @param list
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
  * @param ctx
  *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
@@ -134,6 +141,7 @@ struct mlx5_list {
  */
 __rte_internal
 struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
 				   mlx5_list_create_cb cb_create,
 				   mlx5_list_match_cb cb_match,
 				   mlx5_list_remove_cb cb_remove,
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index da4d2fdadc..ced88f5394 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 						   flow_dv_port_id_create_cb,
 						   flow_dv_port_id_match_cb,
 						   flow_dv_port_id_remove_cb,
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 						    flow_dv_push_vlan_create_cb,
 						    flow_dv_push_vlan_match_cb,
 						    flow_dv_push_vlan_remove_cb,
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 						  flow_dv_sample_remove_cb,
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -1750,7 +1750,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
 				       mlx5_hrxq_clone_cb,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 982056cb41..4f7cdb0622 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -10047,7 +10047,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
 					      flow_dv_matcher_remove_cb,
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index e6176e70d2..a04f93e1d4 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v4 15/26] common/mlx5: call list callbacks with context

[Suanming Mou]

This commit optimizes to call the list callback functions with global
context directly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c |  24 ++---
 drivers/common/mlx5/mlx5_common_utils.h |  36 ++++----
 drivers/net/mlx5/mlx5_flow.h            |  94 +++++++++----------
 drivers/net/mlx5/mlx5_flow_dv.c         | 115 +++++++++++-------------
 drivers/net/mlx5/mlx5_rx.h              |  13 ++-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +++--------
 6 files changed, 132 insertions(+), 203 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 6ac78ba97f..50c98143f7 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -55,7 +55,7 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
+		if (list->cb_match(list->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
@@ -96,7 +96,7 @@ static struct mlx5_list_entry *
 mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
@@ -121,9 +121,9 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
 			if (list->lcores_share)
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			else
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -162,7 +162,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_rwlock_read_unlock(&list->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
+	entry = list->cb_create(list->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
@@ -174,9 +174,9 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list, entry, ctx);
+	local_entry = list->cb_clone(list->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
+		list->cb_remove(list->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
@@ -192,8 +192,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
+			list->cb_remove(list->ctx, entry);
+			list->cb_clone_free(list->ctx, local_entry);
 			return mlx5_list_cache_insert(list, lcore_index, oentry,
 						      ctx);
 		}
@@ -223,9 +223,9 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (list->lcores_share)
-			list->cb_clone_free(list, entry);
+			list->cb_clone_free(list->ctx, entry);
 		else
-			list->cb_remove(list, entry);
+			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
@@ -244,7 +244,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
+		list->cb_remove(list->ctx, gentry);
 		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 000279d236..a691ff8f0d 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -33,19 +33,19 @@ struct mlx5_list_cache {
 /**
  * Type of callback function for entry removal.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance user context.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_remove_cb)(void *tool_ctx,
+				    struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance context.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -54,34 +54,28 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+typedef int (*mlx5_list_match_cb)(void *tool_ctx,
 				   struct mlx5_list_entry *entry, void *ctx);
 
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)(void *tool_ctx,
+				      struct mlx5_list_entry *entry, void *ctx);
 
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
+					struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
+ * @param tool_ctx
+ *   The mlx5 tool instance context.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
+						       void *ctx);
 
 /**
  * Linked mlx5 list structure.
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index ce4d205e86..1d96c61663 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1619,64 +1619,50 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
 
-int flow_dv_matcher_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *ctx);
-struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
-
-int flow_dv_port_id_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused,
-				 void *cb_ctx);
-void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused);
-int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
-				(struct mlx5_list *list,
+int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_matcher_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(void *tool_ctx,
+				struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry);
+
+int flow_dv_push_vlan_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			       void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(void *tool_ctx,
+						    void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb(void *tool_ctx,
 				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
-						 struct mlx5_list_entry *entry,
-						 void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_sample_clone_cb
-				(struct mlx5_list *list,
+void flow_dv_push_vlan_clone_free_cb(void *tool_ctx,
+				     struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_sample_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb(void *tool_ctx,
 				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+void flow_dv_sample_clone_free_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_dest_array_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+
+int flow_dv_dest_array_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+				void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(void *tool_ctx,
+						     void *cb_ctx);
+void flow_dv_dest_array_remove_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(void *tool_ctx,
+				      struct mlx5_list_entry *entry);
+
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 4f7cdb0622..d32dd76c11 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3774,23 +3774,21 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->port_id != res->port_id;
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
@@ -3821,11 +3819,11 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_clone_cb(void *tool_ctx,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
@@ -3843,12 +3841,11 @@ flow_dv_port_id_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
@@ -3893,23 +3890,21 @@ flow_dv_port_id_action_resource_register
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
+			   struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
@@ -3946,11 +3941,11 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_clone_cb(void *tool_ctx,
+			   struct mlx5_list_entry *entry __rte_unused,
+			   void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx;
@@ -3968,12 +3963,11 @@ flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
@@ -9952,7 +9946,7 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 }
 
 static struct mlx5_list_entry *
-flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -9975,7 +9969,7 @@ flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 static void
-flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
@@ -10218,7 +10212,7 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10233,11 +10227,9 @@ flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_matcher_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
 	struct mlx5_flow_dv_matcher *resource;
@@ -10733,7 +10725,7 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
 			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10762,9 +10754,7 @@ flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -10851,7 +10841,7 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
@@ -10877,16 +10867,15 @@ flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry)
+flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			resource->idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
 }
 
 /**
@@ -10929,14 +10918,14 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
 			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_dv_dest_array_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
 	if (ctx_resource->num_of_dest == resource->num_of_dest &&
@@ -10958,9 +10947,7 @@ flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11065,9 +11052,9 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
+			    struct mlx5_list_entry *entry __rte_unused,
+			    void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11093,8 +11080,8 @@ flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
+				 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
 			container_of(entry, typeof(*resource), entry);
@@ -13568,7 +13555,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
 			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_matcher *resource = container_of(entry,
@@ -13710,10 +13697,9 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
 				  container_of(entry, typeof(*resource), entry);
 
@@ -13767,10 +13753,9 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
 			container_of(entry, typeof(*resource), entry);
 
@@ -13839,7 +13824,7 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
@@ -13887,7 +13872,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 5450ddd388..3f2b99fb65 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index aa9e973d10..7893b3edd4 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,25 +2093,10 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Match an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   0 if match, none zero if not match.
- */
 int
-mlx5_hrxq_match_cb(struct mlx5_list *list,
-		   struct mlx5_list_entry *entry,
-		   void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2251,10 +2236,9 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
 	__mlx5_hrxq_remove(dev, hrxq);
@@ -2305,25 +2289,10 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
 	return NULL;
 }
 
-/**
- * Create an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   queue entry on success, NULL otherwise.
- */
 struct mlx5_list_entry *
-mlx5_hrxq_create_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry __rte_unused,
-		    void *cb_ctx)
+mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq;
@@ -2333,11 +2302,10 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-mlx5_hrxq_clone_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry,
+mlx5_hrxq_clone_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		    void *cb_ctx __rte_unused)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	uint32_t hrxq_idx = 0;
@@ -2351,10 +2319,9 @@ mlx5_hrxq_clone_cb(struct mlx5_list *list,
 }
 
 void
-mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
-- 
2.25.1

[dpdk-dev] [PATCH v4 16/26] common/mlx5: add per-lcore cache to hash list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates
all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and
hlist.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 238 +++++++------------
 drivers/common/mlx5/mlx5_common_utils.h | 141 +++---------
 drivers/net/mlx5/linux/mlx5_os.c        |  35 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++++++----
 drivers/net/mlx5/mlx5_flow.h            |  92 ++++----
 drivers/net/mlx5/mlx5_flow_dv.c         | 292 +++++++++++++++---------
 8 files changed, 482 insertions(+), 482 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 50c98143f7..4e385c616a 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -13,25 +13,21 @@
 
 /********************* mlx5 list ************************/
 
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx, bool lcores_share,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
+static int
+mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
+	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       mlx5_list_match_cb cb_match,
+	       mlx5_list_remove_cb cb_remove,
+	       mlx5_list_clone_cb cb_clone,
+	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	struct mlx5_list *list;
 	int i;
 
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
-		return NULL;
+		return -EINVAL;
 	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -45,6 +41,28 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
+	return 0;
+}
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (mlx5_list_init(list, name, ctx, lcores_share,
+			   cb_create, cb_match, cb_remove, cb_clone,
+			   cb_clone_free) != 0) {
+		mlx5_free(list);
+		return NULL;
+	}
 	return list;
 }
 
@@ -254,8 +272,8 @@ mlx5_list_unregister(struct mlx5_list *list,
 	return 1;
 }
 
-void
-mlx5_list_destroy(struct mlx5_list *list)
+static void
+mlx5_list_uninit(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
 	int i;
@@ -266,15 +284,21 @@ mlx5_list_destroy(struct mlx5_list *list)
 			entry = LIST_FIRST(&list->cache[i].h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", list->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			}
 		}
 	}
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	mlx5_list_uninit(list);
 	mlx5_free(list);
 }
 
@@ -287,37 +311,24 @@ mlx5_list_get_entry_num(struct mlx5_list *list)
 
 /********************* Hash List **********************/
 
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
 struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
+		  bool lcores_share, void *ctx, mlx5_list_create_cb cb_create,
+		  mlx5_list_match_cb cb_match,
+		  mlx5_list_remove_cb cb_remove,
+		  mlx5_list_clone_cb cb_clone,
+		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+		DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
+			"be aligned to 0x%" PRIX32 ".", size, act_size);
 	} else {
 		act_size = size;
 	}
@@ -331,61 +342,24 @@ mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 			name ? name : "None");
 		return NULL;
 	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
 	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
+	h->lcores_share = lcores_share;
+	h->direct_key = direct_key;
+	for (i = 0; i < act_size; i++) {
+		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   cb_create, cb_match, cb_remove, cb_clone,
+				   cb_clone_free) != 0) {
+			mlx5_free(h);
+			return NULL;
 		}
 	}
-	return node;
+	DRV_LOG(DEBUG, "Hash list %s with size 0x%" PRIX32 " was created.",
+		name, act_size);
+	return h;
 }
 
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
 
-struct mlx5_hlist_entry *
+struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
@@ -394,102 +368,44 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
+	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
 }
 
-struct mlx5_hlist_entry*
+struct mlx5_list_entry*
 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry;
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
+	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	if (likely(entry)) {
+		if (h->lcores_share)
+			entry->gentry->bucket_idx = idx;
+		else
+			entry->bucket_idx = idx;
 	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
 	return entry;
 }
 
 int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
+	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+							      entry->bucket_idx;
+
+	return mlx5_list_unregister(&h->buckets[idx].l, entry);
 }
 
 void
 mlx5_hlist_destroy(struct mlx5_hlist *h)
 {
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
+	uint32_t i;
 
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
+	for (i = 0; i <= h->mask; i++)
+		mlx5_list_uninit(&h->buckets[i].l);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index a691ff8f0d..4bb974fa3e 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -20,10 +20,13 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t ref_cnt __rte_aligned(8); /* 0 means, entry is invalid. */
 	uint32_t lcore_idx;
-	struct mlx5_list_entry *gentry;
-};
+	union {
+		struct mlx5_list_entry *gentry;
+		uint32_t bucket_idx;
+	};
+} __rte_packed;
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
@@ -212,108 +215,24 @@ __rte_internal
 uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list);
 
-/************************ Hash list *****************************/
-
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
-
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE			32
-
-struct mlx5_hlist;
-
-/**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
- */
-struct mlx5_hlist_entry {
-	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint32_t idx; /* Bucket index the entry belongs to. */
-	uint32_t ref_cnt; /* Reference count. */
-};
-
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
-				     struct mlx5_hlist_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- * @param key
- *   The new entry key.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry,
-				   uint64_t key, void *ctx);
-
-/**
- * Type of function for user defined hash list entry creation.
- *
- * @param list
- *   The hash list.
- * @param key
- *   The key of the new entry.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer to allocated entry on success, NULL otherwise.
- */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
-				  (struct mlx5_hlist *list,
-				   uint64_t key, void *ctx);
+/********************* Hash List **********************/
 
-/* Hash list bucket head. */
+/* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_hlist_head head; /* List head. */
-	rte_rwlock_t lock; /* Bucket lock. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
+	struct mlx5_list l;
 } __rte_cache_aligned;
 
 /**
  * Hash list table structure
  *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
+ * The hash list bucket using the mlx5_list object for managing.
  */
 struct mlx5_hlist {
-	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
-	/**< number of heads, need to be power of 2. */
-	uint32_t table_sz;
-	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
-	/**< mask to get the index of the list heads. */
-	uint32_t mask;
-	bool direct_key; /* Use the new entry key directly as hash index. */
-	bool write_most; /* List mostly used for append new or destroy. */
-	void *ctx;
-	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
-	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
-	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+	uint32_t mask; /* A mask for the bucket index range. */
+	uint8_t flags;
+	bool direct_key; /* Whether to use the key directly as hash index. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
-	/**< list bucket arrays. */
 };
 
 /**
@@ -330,23 +249,33 @@ struct mlx5_hlist {
  *   Heads array size of the hash list.
  * @param entry_size
  *   Entry size to allocate if cb_create not specified.
- * @param flags
- *   The hash list attribute flags.
+ * @param direct key
+ *   Whether to use the key directly as hash index.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
+ * @param ctx
+ *   The hlist instance context.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
  *   Callback function for entry match.
- * @param cb_destroy
- *   Callback function for entry destroy.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @param cb_clone
+ *   Callback function for entry clone.
+ * @param cb_clone_free
+ *   Callback function for entry clone free.
  * @return
  *   Pointer of the hash list table created, NULL on failure.
  */
 __rte_internal
 struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
-				     uint32_t entry_size, uint32_t flags,
-				     mlx5_hlist_create_cb cb_create,
-				     mlx5_hlist_match_cb cb_match,
-				     mlx5_hlist_remove_cb cb_destroy);
+				     bool direct_key, bool lcores_share,
+				     void *ctx, mlx5_list_create_cb cb_create,
+				     mlx5_list_match_cb cb_match,
+				     mlx5_list_remove_cb cb_remove,
+				     mlx5_list_clone_cb cb_clone,
+				     mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
@@ -365,7 +294,7 @@ struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
  *   Pointer of the hlist entry if found, NULL otherwise.
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
 					   void *ctx);
 
 /**
@@ -384,7 +313,7 @@ struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
  *   registered entry on success, NULL otherwise
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
 					     void *ctx);
 
 /**
@@ -399,7 +328,7 @@ struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
  *   0 on entry removed, 1 on entry still referenced.
  */
 __rte_internal
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry);
 
 /**
  * Destroy the hash list table, all the entries already inserted into the lists
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index ced88f5394..468d8173c4 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -261,7 +261,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+	char s[MLX5_NAME_SIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -314,44 +314,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb,
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
+					  false, sh, flow_dv_tag_create_cb,
 					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb);
+					  flow_dv_tag_remove_cb,
+					  flow_dv_tag_clone_cb,
+					  flow_dv_tag_clone_free_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, MLX5_HLIST_WRITE_MOST |
-					    MLX5_HLIST_DIRECT_KEY,
+					    true, false, sh,
 					    flow_dv_modify_create_cb,
 					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb);
+					    flow_dv_modify_remove_cb,
+					    flow_dv_modify_clone_cb,
+					    flow_dv_modify_clone_free_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, MLX5_HLIST_DIRECT_KEY |
-					      MLX5_HLIST_WRITE_MOST,
+					      true, true, sh,
 					      flow_dv_encap_decap_create_cb,
 					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb);
+					      flow_dv_encap_decap_remove_cb,
+					      flow_dv_encap_decap_clone_cb,
+					     flow_dv_encap_decap_clone_free_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -1783,15 +1783,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
-						      0, 0,
+						      false, true, eth_dev,
 						      flow_dv_mreg_create_cb,
 						      flow_dv_mreg_match_cb,
-						      flow_dv_mreg_remove_cb);
+						      flow_dv_mreg_remove_cb,
+						      flow_dv_mreg_clone_cb,
+						    flow_dv_mreg_clone_free_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
-		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 6661d041ed..2d4330198b 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1356,20 +1356,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 5da3d93a5b..56ac018c7e 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 8e3af5bccb..367fbc8e4a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3983,28 +3983,27 @@ flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
-flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
-		      struct mlx5_hlist_entry *entry,
-		      uint64_t key, void *cb_ctx __rte_unused)
+flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
+		      struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
 
-	return mcp_res->mark_id != key;
+	return mcp_res->mark_id != *(uint32_t *)(ctx->data);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
-		       void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
 	struct rte_flow_error *error = ctx->error;
 	uint32_t idx = 0;
 	int ret;
-	uint32_t mark_id = key;
+	uint32_t mark_id = *(uint32_t *)(ctx->data);
 	struct rte_flow_attr attr = {
 		.group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
 		.ingress = 1,
@@ -4110,6 +4109,36 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return &mcp_res->hlist_ent;
 }
 
+struct mlx5_list_entry *
+flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	uint32_t idx = 0;
+
+	mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
+	if (!mcp_res) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	memcpy(mcp_res, oentry, sizeof(*mcp_res));
+	mcp_res->idx = idx;
+	return &mcp_res->hlist_ent;
+}
+
+void
+flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_mreg_copy_resource *mcp_res =
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
+}
+
 /**
  * Add a flow of copying flow metadata registers in RX_CP_TBL.
  *
@@ -4140,10 +4169,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 			  struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
+		.data = &mark_id,
 	};
 
 	/* Check if already registered. */
@@ -4156,11 +4186,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 }
 
 void
-flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
+flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
-	struct rte_eth_dev *dev = list->ctx;
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
@@ -4206,14 +4236,17 @@ flow_mreg_del_copy_action(struct rte_eth_dev *dev,
 static void
 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
 {
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check if default flow is registered. */
 	if (!priv->mreg_cp_tbl)
 		return;
-	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
-				  MLX5_DEFAULT_COPY_ID, NULL);
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
 	if (!entry)
 		return;
 	mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
@@ -4239,6 +4272,8 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check whether extensive metadata feature is engaged. */
 	if (!priv->config.dv_flow_en ||
@@ -4250,9 +4285,11 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 	 * Add default mreg copy flow may be called multiple time, but
 	 * only be called once in stop. Avoid register it twice.
 	 */
-	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
 		return 0;
-	mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
+	mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
 	if (!mcp_res)
 		return -rte_errno;
 	return 0;
@@ -8367,7 +8404,7 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	union tunnel_offload_mark mbits = { .val = mark };
 	union mlx5_flow_tbl_key table_key = {
 		{
@@ -8379,16 +8416,20 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 			.is_egress = 0,
 		}
 	};
-	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &table_key.v64,
+	};
+
+	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
 	return he ?
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
 
 static void
-mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry)
+mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
@@ -8397,26 +8438,26 @@ mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
 }
 
 static int
-mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-				  struct mlx5_hlist_entry *entry,
-				  uint64_t key, void *cb_ctx __rte_unused)
+mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
 }
 
-static struct mlx5_hlist_entry *
-mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
-				   void *ctx __rte_unused)
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 
 	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
@@ -8445,13 +8486,36 @@ mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return NULL;
 }
 
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *oentry,
+				  void *cb_ctx __rte_unused)
+{
+	struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
+						   0, SOCKET_ID_ANY);
+
+	if (!tte)
+		return NULL;
+	memcpy(tte, oentry, sizeof(*tte));
+	return &tte->hash;
+}
+
+static void
+mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
+				       struct mlx5_list_entry *entry)
+{
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	mlx5_free(tte);
+}
+
 static uint32_t
 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 				const struct mlx5_flow_tunnel *tunnel,
 				uint32_t group, uint32_t *table,
 				struct rte_flow_error *error)
 {
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key key = {
 		.tunnel_id = tunnel ? tunnel->tunnel_id : 0,
@@ -8459,9 +8523,12 @@ tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 	};
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_hlist *group_hash;
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &key.val,
+	};
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_register(group_hash, key.val, NULL);
+	he = mlx5_hlist_register(group_hash, key.val, &ctx);
 	if (!he)
 		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
@@ -8575,15 +8642,17 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
-					   mlx5_flow_tunnel_grp2tbl_remove_cb);
+					   mlx5_flow_tunnel_grp2tbl_remove_cb,
+					   mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(ipool, id);
 		return NULL;
 	}
-	tunnel->groups->ctx = priv->sh;
 	/* initiate new PMD tunnel */
 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
 	tunnel->tunnel_id = id;
@@ -8683,15 +8752,17 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES), 0,
-					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
+					 rte_align32pow2(MLX5_MAX_TABLES),
+					 false, true, sh,
+					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-					 mlx5_flow_tunnel_grp2tbl_remove_cb);
+					 mlx5_flow_tunnel_grp2tbl_remove_cb,
+					 mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
 	}
-	thub->groups->ctx = sh;
 	sh->tunnel_hub = thub;
 
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 1d96c61663..98e90fec0d 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -480,7 +480,7 @@ struct mlx5_flow_dv_matcher {
 
 /* Encap/decap resource structure. */
 struct mlx5_flow_dv_encap_decap_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/* Pointer to next element. */
 	uint32_t refcnt; /**< Reference counter. */
 	void *action;
@@ -495,7 +495,7 @@ struct mlx5_flow_dv_encap_decap_resource {
 
 /* Tag resource structure. */
 struct mlx5_flow_dv_tag_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry for tag resource, tag value as the key. */
 	void *action;
 	/**< Tag action object. */
@@ -519,7 +519,7 @@ struct mlx5_flow_dv_tag_resource {
 
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
@@ -569,7 +569,7 @@ struct mlx5_flow_mreg_copy_resource {
 	 *  - Key is 32/64-bit MARK action ID.
 	 *  - MUST be the first entry.
 	 */
-	struct mlx5_hlist_entry hlist_ent;
+	struct mlx5_list_entry hlist_ent;
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
 	uint32_t idx;
@@ -586,7 +586,7 @@ struct mlx5_flow_tbl_tunnel_prm {
 
 /* Table data structure of the hash organization. */
 struct mlx5_flow_tbl_data_entry {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
@@ -926,7 +926,7 @@ struct mlx5_flow_tunnel_hub {
 
 /* convert jump group to flow table ID in tunnel rules */
 struct tunnel_tbl_entry {
-	struct mlx5_hlist_entry hash;
+	struct mlx5_list_entry hash;
 	uint32_t flow_table;
 	uint32_t tunnel_id;
 	uint32_t group;
@@ -1573,51 +1573,59 @@ int mlx5_action_handle_flush(struct rte_eth_dev *dev);
 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id);
 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh);
 
-/* Hash list callbacks for flow tables: */
-struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *entry_ctx);
-int flow_dv_tbl_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tbl_create_cb(void *tool_ctx, void *entry_ctx);
+int flow_dv_tbl_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
+void flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tbl_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *entry_ctx);
+void flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
 struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 		uint32_t table_level, uint8_t egress, uint8_t transfer,
 		bool external, const struct mlx5_flow_tunnel *tunnel,
 		uint32_t group_id, uint8_t dummy,
 		uint32_t table_id, struct rte_flow_error *error);
 
-struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *cb_ctx);
-int flow_dv_tag_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx);
+int flow_dv_tag_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
-
-int flow_dv_modify_match_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry,
-			    uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
-						  uint64_t key, void *ctx);
-void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry);
-
-struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
-						uint64_t key, void *ctx);
-int flow_dv_mreg_match_cb(struct mlx5_hlist *list,
-			  struct mlx5_hlist_entry *entry, uint64_t key,
+void flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tag_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *cb_ctx);
+void flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_modify_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_modify_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_modify_clone_cb(void *tool_ctx,
+						struct mlx5_list_entry *oentry,
+						void *ctx);
+void flow_dv_modify_clone_free_cb(void *tool_ctx,
+				  struct mlx5_list_entry *entry);
+
+struct mlx5_list_entry *flow_dv_mreg_create_cb(void *tool_ctx, void *ctx);
+int flow_dv_mreg_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			  void *cb_ctx);
-void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry);
-
-int flow_dv_encap_decap_match_cb(struct mlx5_hlist *list,
-				 struct mlx5_hlist_entry *entry,
-				 uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-				uint64_t key, void *cb_ctx);
-void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry);
+void flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_mreg_clone_cb(void *tool_ctx,
+					      struct mlx5_list_entry *entry,
+					      void *ctx);
+void flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_encap_decap_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+				 void *cb_ctx);
+struct mlx5_list_entry *flow_dv_encap_decap_create_cb(void *tool_ctx,
+						      void *cb_ctx);
+void flow_dv_encap_decap_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_encap_decap_clone_cb(void *tool_ctx,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_encap_decap_clone_free_cb(void *tool_ctx,
+				       struct mlx5_list_entry *entry);
 
 int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			     void *ctx);
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index d32dd76c11..9116f8b9d4 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3580,25 +3580,9 @@ flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
 	return 0;
 }
 
-/**
- * Match encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
 int
-flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
-			     struct mlx5_hlist_entry *entry,
-			     uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3617,25 +3601,10 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 	return -1;
 }
 
-/**
- * Allocate encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
-struct mlx5_hlist_entry *
-flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-			      uint64_t key __rte_unused,
-			      void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3673,6 +3642,38 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			     void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	uint32_t idx;
+
+	cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+					   &idx);
+	if (!cache_resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(cache_resource, oentry, sizeof(*cache_resource));
+	cache_resource->idx = idx;
+	return &cache_resource->entry;
+}
+
+void
+flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *res =
+				       container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
+}
+
 /**
  * Find existing encap/decap resource or create and register a new one.
  *
@@ -3697,7 +3698,7 @@ flow_dv_encap_decap_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	union {
 		struct {
 			uint32_t ft_type:8;
@@ -5322,30 +5323,14 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
 	return ret;
 }
 
-/**
- * Match modify-header resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx
- *   Pointer to new modify-header resource.
- *
- * @return
- *   0 on matching, non-zero otherwise.
- */
 int
-flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
-			struct mlx5_hlist_entry *entry,
-			uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	struct mlx5_flow_dv_modify_hdr_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
 
 	key_len += ref->actions_num * sizeof(ref->actions[0]);
@@ -5353,11 +5338,10 @@ flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
-			 void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
@@ -5396,6 +5380,33 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
 	return &entry->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *oentry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *entry;
+	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+
+	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	return &entry->entry;
+}
+
+void
+flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 /**
  * Validate the sample action.
  *
@@ -5667,7 +5678,7 @@ flow_dv_modify_hdr_resource_register
 	uint32_t key_len = sizeof(*resource) -
 			   offsetof(typeof(*resource), ft_type) +
 			   resource->actions_num * sizeof(resource->actions[0]);
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
 		.data = resource,
@@ -9975,16 +9986,16 @@ flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
 	mlx5_free(entry);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
-	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data;
+	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
 	struct rte_flow_error *error = ctx->error;
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
 	struct mlx5_flow_tbl_resource *tbl;
 	void *domain;
 	uint32_t idx = 0;
@@ -10061,13 +10072,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 }
 
 int
-flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key64,
-		     void *cb_ctx __rte_unused)
+flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
 		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
 
 	return tbl_data->level != key.level ||
 	       tbl_data->id != key.id ||
@@ -10076,6 +10087,39 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
 	       tbl_data->is_egress != !!key.is_egress;
 }
 
+struct mlx5_list_entry *
+flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data;
+	struct rte_flow_error *error = ctx->error;
+	uint32_t idx = 0;
+
+	tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
+	if (!tbl_data) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate flow table data entry");
+		return NULL;
+	}
+	memcpy(tbl_data, oentry, sizeof(*tbl_data));
+	tbl_data->idx = idx;
+	return &tbl_data->entry;
+}
+
+void
+flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data =
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
+}
+
 /**
  * Get a flow table.
  *
@@ -10126,9 +10170,10 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = &tt_prm,
+		.data = &table_key.v64,
+		.data2 = &tt_prm,
 	};
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 
 	entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
@@ -10147,12 +10192,11 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
-		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
 
 	MLX5_ASSERT(entry && sh);
 	if (tbl_data->jump.action)
@@ -10160,7 +10204,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 	if (tbl_data->tbl.obj)
 		mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
 	if (tbl_data->tunnel_offload && tbl_data->external) {
-		struct mlx5_hlist_entry *he;
+		struct mlx5_list_entry *he;
 		struct mlx5_hlist *tunnel_grp_hash;
 		struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
 		union tunnel_tbl_key tunnel_key = {
@@ -10169,11 +10213,14 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			.group = tbl_data->group_id
 		};
 		uint32_t table_level = tbl_data->level;
+		struct mlx5_flow_cb_ctx ctx = {
+			.data = (void *)&tunnel_key.val,
+		};
 
 		tunnel_grp_hash = tbl_data->tunnel ?
 					tbl_data->tunnel->groups :
 					thub->groups;
-		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
+		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
 		if (he)
 			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
@@ -10326,29 +10373,29 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 	return 0;
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
+struct mlx5_list_entry *
+flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct rte_flow_error *error = ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_tag_resource *entry;
 	uint32_t idx = 0;
 	int ret;
 
 	entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
 	if (!entry) {
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
 	entry->idx = idx;
-	entry->tag_id = key;
-	ret = mlx5_flow_os_create_flow_action_tag(key,
+	entry->tag_id = *(uint32_t *)(ctx->data);
+	ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
 						  &entry->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create action");
 		return NULL;
@@ -10357,14 +10404,45 @@ flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
 }
 
 int
-flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key,
-		     void *cb_ctx __rte_unused)
+flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
-	return key != tag->tag_id;
+	return *(uint32_t *)(ctx->data) != tag->tag_id;
+}
+
+struct mlx5_list_entry *
+flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		     void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_tag_resource *entry;
+	uint32_t idx = 0;
+
+	entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate tag resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry));
+	entry->idx = idx;
+	return &entry->entry;
+}
+
+void
+flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_tag_resource *tag =
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
 }
 
 /**
@@ -10391,9 +10469,13 @@ flow_dv_tag_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_dv_tag_resource *resource;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_cb_ctx ctx = {
+					.error = error,
+					.data = &tag_be24,
+					};
 
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
+	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
@@ -10405,12 +10487,11 @@ flow_dv_tag_resource_register
 }
 
 void
-flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
 	MLX5_ASSERT(tag && sh && tag->action);
 	claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
@@ -13592,19 +13673,10 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Release encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- */
 void
-flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry)
+flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
 				       container_of(entry, typeof(*res), entry);
 
@@ -13664,8 +13736,8 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused,
-			 struct mlx5_hlist_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
-- 
2.25.1

[dpdk-dev] [PATCH v4 17/26] common/mlx5: allocate cache list memory individually

[Suanming Mou]

Currently, the list's local cache instance memory is allocated with
the list. As the local cache instance array size is RTE_MAX_LCORE,
most of the cases the system will only have very limited cores.
allocate the instance memory individually per core will be more
economic to the memory.

This commit changes the instance array to pointer array, allocate
the local cache memory only when the core is to be used.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 4e385c616a..f75b1cb0da 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 4bb974fa3e..d49fb64457 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-- 
2.25.1

[dpdk-dev] [PATCH v4 18/26] common/mlx5: optimize cache list object memory

[Suanming Mou]

Currently, hash list uses the cache list as bucket list. The list
in the buckets have the same name, ctx and callbacks. This wastes
the memory.

This commit abstracts all the name, ctx and callback members in the
list to a constant struct and others to the inconstant struct, uses
the wrapper functions to satisfy both hash list and cache list can
set the list constant and inconstant struct individually.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index f75b1cb0da..858c8d8164 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index d49fb64457..5718a21be0 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -80,6 +80,32 @@ typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
 typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
 						       void *ctx);
 
+/**
+ * Linked mlx5 list constant object.
+ */
+struct mlx5_list_const {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	void *ctx; /* user objects target to callback. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	/**< entry clone free callback. */
+};
+
+/**
+ * Linked mlx5 list inconstant data.
+ */
+struct mlx5_list_inconst {
+	rte_rwlock_t lock; /* read/write lock. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+};
+
 /**
  * Linked mlx5 list structure.
  *
@@ -96,19 +122,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
  *
  */
 struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	void *ctx; /* user objects target to callback. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-	volatile uint32_t gen_cnt; /* List modification may update it. */
-	volatile uint32_t count; /* number of entries in list. */
-	rte_rwlock_t lock; /* read/write lock. */
+	struct mlx5_list_const l_const;
+	struct mlx5_list_inconst l_inconst;
 };
 
 /**
@@ -219,7 +234,7 @@ mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_list l;
+	struct mlx5_list_inconst l;
 } __rte_cache_aligned;
 
 /**
@@ -231,7 +246,7 @@ struct mlx5_hlist {
 	uint32_t mask; /* A mask for the bucket index range. */
 	uint8_t flags;
 	bool direct_key; /* Whether to use the key directly as hash index. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
+	struct mlx5_list_const l_const; /* List constant data. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v4 19/26] common/mlx5: support list non-lcore operations

[Suanming Mou]

This commit supports the list non-lcore operations with
an extra sub-list and lock.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 858c8d8164..c8b48f3afc 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.lcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.lcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
+	return ret;
 }
 
 void
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 5718a21be0..613d29de0c 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t lcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v4 20/26] net/mlx5: move modify header allocator to ipool

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Modify header actions are allocated by mlx5_malloc which has a big
overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has
its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in
the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has
better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header
handler.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 2d4330198b..499d8449ac 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -799,6 +799,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -812,6 +813,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 56ac018c7e..d88abef12c 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1123,6 +1136,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 98e90fec0d..da6367c212 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -504,23 +504,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 9116f8b9d4..9666c11861 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5338,6 +5338,45 @@ flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
+static struct mlx5_indexed_pool *
+flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
+{
+	struct mlx5_indexed_pool *ipool = __atomic_load_n
+				     (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
+
+	if (!ipool) {
+		struct mlx5_indexed_pool *expected = NULL;
+		struct mlx5_indexed_pool_config cfg =
+		    (struct mlx5_indexed_pool_config) {
+		       .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
+								   (index + 1) *
+					   sizeof(struct mlx5_modification_cmd),
+		       .trunk_size = 64,
+		       .grow_trunk = 3,
+		       .grow_shift = 2,
+		       .need_lock = 1,
+		       .release_mem_en = 1,
+		       .malloc = mlx5_malloc,
+		       .free = mlx5_free,
+		       .type = "mlx5_modify_action_resource",
+		};
+
+		cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
+		ipool = mlx5_ipool_create(&cfg);
+		if (!ipool)
+			return NULL;
+		if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
+						 &expected, ipool, false,
+						 __ATOMIC_SEQ_CST,
+						 __ATOMIC_SEQ_CST)) {
+			mlx5_ipool_destroy(ipool);
+			ipool = __atomic_load_n(&sh->mdh_ipools[index],
+						__ATOMIC_SEQ_CST);
+		}
+	}
+	return ipool;
+}
+
 struct mlx5_list_entry *
 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
@@ -5346,12 +5385,20 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
+							  ref->actions_num - 1);
 	int ret;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
+	uint32_t idx;
 
-	entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
-			    SOCKET_ID_ANY);
+	if (unlikely(!ipool)) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL, "cannot allocate modify ipool");
+		return NULL;
+	}
+	entry = mlx5_ipool_zmalloc(ipool, &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5371,25 +5418,29 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 					(sh->ctx, ns, entry,
 					 data_len, &entry->action);
 	if (ret) {
-		mlx5_free(entry);
+		mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create modification action");
 		return NULL;
 	}
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 struct mlx5_list_entry *
-flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
-			struct mlx5_list_entry *oentry, void *cb_ctx)
+flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			void *cb_ctx)
 {
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+	uint32_t idx;
 
-	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
+				  &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5397,14 +5448,18 @@ flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
 		return NULL;
 	}
 	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 void
-flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	mlx5_free(entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *res =
+		container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
@@ -13736,14 +13791,14 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
-			 struct mlx5_list_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
-	mlx5_free(entry);
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v4 21/26] net/mlx5: adjust the hash bucket size

[Suanming Mou]

With the new per core optimization to the list, the hash bucket size
can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 468d8173c4..cfb33251cf 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 499d8449ac..08e795ea4c 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -373,7 +373,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 906aa43c5a..ca67ce8213 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 367fbc8e4a..b92ebe9029 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -8642,7 +8642,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8751,8 +8751,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-- 
2.25.1

[dpdk-dev] [PATCH v4 22/26] net/mlx5: enable index pool per-core cache

[Suanming Mou]

This commit enables the tag and header modify action index pool
per-core cache in non-reclaim memory mode.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 08e795ea4c..53dee9164a 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -214,7 +214,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1126,6 +1127,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index d88abef12c..2e2a98345f 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1093,6 +1093,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 9666c11861..b30e5a82ce 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5355,7 +5355,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
-- 
2.25.1

[dpdk-dev] [PATCH v4 23/26] net/mlx5: optimize hash list table allocate on demand

[Suanming Mou]

Currently, all the hash list tables are allocated during start up.
Since different applications may only use dedicated limited actions,
optimized the hash list table allocate on demand will save initial
memory.

This commit optimizes hash list table allocate on demand.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index cfb33251cf..38c5672436 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index ca67ce8213..fe86bb40d3 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index b30e5a82ce..9fc50a5daa 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -310,6 +310,41 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 	}
 }
 
+static inline struct mlx5_hlist *
+flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
+		     const char *name, uint32_t size, bool direct_key,
+		     bool lcores_share, void *ctx,
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove,
+		     mlx5_list_clone_cb cb_clone,
+		     mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_hlist *hl;
+	struct mlx5_hlist *expected = NULL;
+	char s[MLX5_NAME_SIZE];
+
+	hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	if (likely(hl))
+		return hl;
+	snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
+	hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
+			ctx, cb_create, cb_match, cb_remove, cb_clone,
+			cb_clone_free);
+	if (!hl) {
+		DRV_LOG(ERR, "%s hash creation failed", name);
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
+					 __ATOMIC_SEQ_CST,
+					 __ATOMIC_SEQ_CST)) {
+		mlx5_hlist_destroy(hl);
+		hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	}
+	return hl;
+}
+
 /* Update VLAN's VID/PCP based on input rte_flow_action.
  *
  * @param[in] action
@@ -3724,8 +3759,20 @@ flow_dv_encap_decap_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *encaps_decaps;
 	uint64_t key64;
 
+	encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
+				"encaps_decaps",
+				MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
+				true, true, sh,
+				flow_dv_encap_decap_create_cb,
+				flow_dv_encap_decap_match_cb,
+				flow_dv_encap_decap_remove_cb,
+				flow_dv_encap_decap_clone_cb,
+				flow_dv_encap_decap_clone_free_cb);
+	if (unlikely(!encaps_decaps))
+		return -rte_errno;
 	resource->flags = dev_flow->dv.group ? 0 : 1;
 	key64 =  __rte_raw_cksum(&encap_decap_key.v32,
 				 sizeof(encap_decap_key.v32), 0);
@@ -3733,7 +3780,7 @@ flow_dv_encap_decap_resource_register
 	    MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
 	    resource->size)
 		key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
-	entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
+	entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -5739,8 +5786,20 @@ flow_dv_modify_hdr_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *modify_cmds;
 	uint64_t key64;
 
+	modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
+				"hdr_modify",
+				MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
+				true, false, sh,
+				flow_dv_modify_create_cb,
+				flow_dv_modify_match_cb,
+				flow_dv_modify_remove_cb,
+				flow_dv_modify_clone_cb,
+				flow_dv_modify_clone_free_cb);
+	if (unlikely(!modify_cmds))
+		return -rte_errno;
 	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
 								resource->root))
@@ -5748,7 +5807,7 @@ flow_dv_modify_hdr_resource_register
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
 	key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
-	entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
+	entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10530,8 +10589,20 @@ flow_dv_tag_resource_register
 					.error = error,
 					.data = &tag_be24,
 					};
-
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
+	struct mlx5_hlist *tag_table;
+
+	tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
+				      "tags",
+				      MLX5_TAGS_HLIST_ARRAY_SIZE,
+				      false, false, priv->sh,
+				      flow_dv_tag_create_cb,
+				      flow_dv_tag_match_cb,
+				      flow_dv_tag_remove_cb,
+				      flow_dv_tag_clone_cb,
+				      flow_dv_tag_clone_free_cb);
+	if (unlikely(!tag_table))
+		return -rte_errno;
+	entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index a04f93e1d4..5da362a9d5 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
-- 
2.25.1

[dpdk-dev] [PATCH v4 24/26] net/mlx5: change memory release configuration

[Suanming Mou]

This commit changes the index pool memory release configuration
to 0 when memory reclaim mode is not required.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 53dee9164a..10594f5961 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -795,6 +795,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
-- 
2.25.1

[dpdk-dev] [PATCH v4 25/26] net/mlx5: optimize Rx queue match

[Suanming Mou]

As hrxq struct has the indirect table pointer, while matching the
hrxq, better to use the hrxq indirect table instead of searching
from the list.

This commit optimizes the hrxq indirect table matching.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 7893b3edd4..23685d7654 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v4 26/26] doc: add mlx5 multiple-thread flow insertion optimization

[Suanming Mou]

This commit adds the multiple-thread flow insertion optimization
description.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/nics/mlx5.rst               | 5 +++++
 doc/guides/rel_notes/release_21_08.rst | 6 ++++++
 2 files changed, 11 insertions(+)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index a16af32e67..d7d7ba9741 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -445,6 +445,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index 0a05cb02fa..d187edc303 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -68,6 +68,12 @@ New Features
   usecases. Configuration happens via standard rawdev enq/deq operations. See
   the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver.
 
+* **Updated Mellanox mlx5 driver.**
+
+  Updated the Mellanox mlx5 driver with new features and improvements, including:
+
+  * Optimize multiple-thread flow insertion rate.
+
 
 Removed Items
 -------------
-- 
2.25.1

[dpdk-dev] [PATCH v5 00/26] net/mlx5: insertion rate optimization

[Suanming Mou]

This patch series optimize the flow insertion rate with adding
local cache to index pool and list.

For object which wants efficient index allocate and free, local
cache will be very helpful.

For index pool, two level cache is added, one as local and another
as global. The global cache is able to save all the allocated
index. That means all the allocated index will not be freed. Once
the local cache is full, the extra index will be flushed to the
global cache. Once local cache is empty, first try to fetch more
index from global, if global is still empty, allocate new trunk
and more index.

For list, sub local core list is introduced. The allocated objects
will be added and released only from local list without any locks.
Only the objects need to be shared will be synced from global list.

---

v5: code rebase to the latest version.

v4:
   - split the list utility to commom patch
   - fix one ipool crash issue in mlx5_ipool_get_cache() function.
   - reorganize the common and mlx5 patches.
   - split the doc update to an individual patch.
   - update and rebase to the latest version.

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
  net/mlx5: optimize modify header action memory
  net/mlx5: remove cache term from the list utility
  net/mlx5: add per lcore cache to the list utility
  net/mlx5: minimize list critical sections
  net/mlx5: manage list cache entries release
  net/mlx5: relax the list utility atomic operations
  net/mlx5: allocate list memory by the create API
  common/mlx5: add per-lcore cache to hash list utility
  net/mlx5: move modify header allocator to ipool

Suanming Mou (17):
  net/mlx5: allow limiting the index pool maximum index
  net/mlx5: add indexed pool local cache
  net/mlx5: add index pool foreach define
  net/mlx5: support index pool non-lcore operations
  net/mlx5: replace flow list with index pool
  common/mlx5: move list utility to common
  common/mlx5: add list lcore share
  common/mlx5: call list callbacks with context
  common/mlx5: allocate cache list memory individually
  common/mlx5: optimize cache list object memory
  common/mlx5: support list non-lcore operations
  net/mlx5: adjust the hash bucket size
  net/mlx5: enable index pool per-core cache
  net/mlx5: optimize hash list table allocate on demand
  net/mlx5: change memory release configuration
  net/mlx5: optimize Rx queue match
  doc: add mlx5 multiple-thread flow insertion optimization

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    1 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  294 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  308 +++---
 drivers/net/mlx5/mlx5_flow.h            |  209 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1206 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  627 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2543 insertions(+), 1414 deletions(-)

-- 
2.25.1

[dpdk-dev] [PATCH v5 01/26] net/mlx5: allow limiting the index pool maximum index

[Suanming Mou]

Some ipool instances in the driver are used as ID\index allocator and
added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value.
The ipool will ensure that no index bigger than the maximum value is
provided.

Use this configuration in ID allocator cases instead of the current
logics. This patch add the maximum ID configurable for the index pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 18fe23e4fb..bf2b2ebc72 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index b54517c6df..15870e14c2 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
-- 
2.25.1

[dpdk-dev] [PATCH v5 02/26] net/mlx5: add indexed pool local cache

[Suanming Mou]

For object which wants efficient index allocate and free, local
cache will be very helpful.

Two level cache is introduced to allocate and free the index more
efficient. One as local and the other as global. The global cache
is able to save all the allocated index. That means all the allocated
index will not be freed. Once the local cache is full, the extra
index will be flushed to the global cache. Once local cache is empty,
first try to fetch more index from global, if global is still empty,
allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index bf2b2ebc72..215024632d 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 15870e14c2..0469062695 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -209,6 +209,11 @@ struct mlx5_indexed_pool_config {
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
 	uint32_t max_idx; /* The maximum index can be allocated. */
+	uint32_t per_core_cache;
+	/*
+	 * Cache entry number per core for performance. Should not be
+	 * set with release_mem_en.
+	 */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
@@ -225,14 +230,39 @@ struct mlx5_indexed_trunk {
 	uint8_t data[] __rte_cache_aligned; /* Entry data start. */
 };
 
+struct mlx5_indexed_cache {
+	struct mlx5_indexed_trunk **trunks;
+	volatile uint32_t n_trunk_valid; /* Trunks allocated. */
+	uint32_t n_trunk; /* Trunk pointer array size. */
+	uint32_t ref_cnt;
+	uint32_t len;
+	uint32_t idx[];
+};
+
+struct mlx5_ipool_per_lcore {
+	struct mlx5_indexed_cache *lc;
+	uint32_t len; /**< Current cache count. */
+	uint32_t idx[]; /**< Cache objects. */
+};
+
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
-	rte_spinlock_t lock; /* Pool lock for multiple thread usage. */
-	uint32_t n_trunk_valid; /* Trunks allocated. */
-	uint32_t n_trunk; /* Trunk pointer array size. */
+	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
 	/* Dim of trunk pointer array. */
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t free_list; /* Index to first free trunk. */
+	union {
+		struct {
+			uint32_t n_trunk_valid; /* Trunks allocated. */
+			uint32_t n_trunk; /* Trunk pointer array size. */
+			struct mlx5_indexed_trunk **trunks;
+			uint32_t free_list; /* Index to first free trunk. */
+		};
+		struct {
+			struct mlx5_indexed_cache *gc;
+			/* Global cache. */
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			/* Local cache. */
+		};
+	};
 #ifdef POOL_DEBUG
 	uint32_t n_entry;
 	uint32_t trunk_new;
@@ -542,6 +572,30 @@ int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool);
  */
 void mlx5_ipool_dump(struct mlx5_indexed_pool *pool);
 
+/**
+ * This function flushes all the cache index back to pool trunk.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ *
+ */
+
+void mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool);
+
+/**
+ * This function gets the available entry from pos.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ * @param pos
+ *   Pointer to the index position start from.
+ *
+ * @return
+ *  - Pointer to the next available entry.
+ *
+ */
+void *mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos);
+
 /**
  * This function allocates new empty Three-level table.
  *
-- 
2.25.1

[dpdk-dev] [PATCH v5 03/26] net/mlx5: add index pool foreach define

[Suanming Mou]

In some cases, application may want to know all the allocated
index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach
operation.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 96 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 +++
 2 files changed, 104 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 215024632d..32f8d65073 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -529,6 +529,16 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -839,6 +849,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 0469062695..737dd7052d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
-- 
2.25.1

[dpdk-dev] [PATCH v5 04/26] net/mlx5: support index pool non-lcore operations

[Suanming Mou]

This commit supports the index pool non-lcore operations with
an extra cache and lcore lock.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 32f8d65073..f9557c09ff 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -275,6 +275,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->lcore_lock);
 	return pool;
 }
 
@@ -515,20 +516,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -549,15 +544,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -570,29 +577,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -635,6 +653,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -814,7 +847,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -883,7 +916,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 737dd7052d..a509b0a4eb 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t lcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
-- 
2.25.1

[dpdk-dev] [PATCH v5 05/26] net/mlx5: replace flow list with index pool

[Suanming Mou]

The flow list is used to save the create flows and to be used only
when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to
flush all the allocated flows.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 ++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 152 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 127 insertions(+), 112 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index be22d9cbd2..867f450cab 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -832,6 +870,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1575,7 +1614,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1611,6 +1649,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 44fbc2da83..b5c05b12e8 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -325,7 +325,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -793,8 +794,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1529,7 +1532,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index f864c1d701..facab913bf 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1367,7 +1375,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1627,7 +1636,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 2feddb0254..d62361362e 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3109,31 +3109,6 @@ mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
 					 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
 }
 
-/**
- * Release resource related QUEUE/RSS action split.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Flow to release id's from.
- */
-static void
-flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
-			     struct rte_flow *flow)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t handle_idx;
-	struct mlx5_flow_handle *dev_handle;
-
-	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
-		       handle_idx, dev_handle, next)
-		if (dev_handle->split_flow_id &&
-		    !dev_handle->is_meter_flow_id)
-			mlx5_ipool_free(priv->sh->ipool
-					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
-					dev_handle->split_flow_id);
-}
-
 static int
 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
 		   const struct rte_flow_attr *attr __rte_unused,
@@ -3429,7 +3404,6 @@ flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 	const struct mlx5_flow_driver_ops *fops;
 	enum mlx5_flow_drv_type type = flow->drv_type;
 
-	flow_mreg_split_qrss_release(dev, flow);
 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
 	fops = flow_get_drv_ops(type);
 	fops->destroy(dev, flow);
@@ -4055,14 +4029,14 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
 
 /* Declare flow create/destroy prototype in advance. */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action actions[],
 		 bool external, struct rte_flow_error *error);
 
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
@@ -4184,8 +4158,8 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	 * be applied, removed, deleted in ardbitrary order
 	 * by list traversing.
 	 */
-	mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
-					 actions, false, error);
+	mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
+					&attr, items, actions, false, error);
 	if (!mcp_res->rix_flow) {
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
 		return NULL;
@@ -4247,7 +4221,7 @@ flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
-	flow_list_destroy(dev, NULL, mcp_res->rix_flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
 }
 
@@ -6187,7 +6161,7 @@ flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
  *   A flow index on success, 0 otherwise and rte_errno is set.
  */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action original_actions[],
@@ -6255,7 +6229,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 				external, hairpin_flow, error);
 	if (ret < 0)
 		goto error_before_hairpin_split;
-	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
+	flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
 	if (!flow) {
 		rte_errno = ENOMEM;
 		goto error_before_hairpin_split;
@@ -6385,12 +6359,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 		if (ret < 0)
 			goto error;
 	}
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
-			     flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
+	flow->type = type;
 	flow_rxq_flags_set(dev, flow);
 	rte_free(translated_actions);
 	tunnel = flow_tunnel_from_rule(wks->flows);
@@ -6412,7 +6381,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_ipool_get
 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
 			rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
+	mlx5_ipool_free(priv->flows[type], idx);
 	rte_errno = ret; /* Restore rte_errno. */
 	ret = rte_errno;
 	rte_errno = ret;
@@ -6464,10 +6433,9 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
 			.type = RTE_FLOW_ACTION_TYPE_END,
 		},
 	};
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow_error error;
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
 						   &attr, &pattern,
 						   actions, false, &error);
 }
@@ -6519,8 +6487,6 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
 	/*
 	 * If the device is not started yet, it is not allowed to created a
 	 * flow from application. PMD default flows and traffic control flows
@@ -6536,8 +6502,9 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		return NULL;
 	}
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
-				  attr, items, actions, true, error);
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
+						   attr, items, actions,
+						   true, error);
 }
 
 /**
@@ -6545,24 +6512,19 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list. If this parameter NULL,
- *   there is no flow removal from the list. Be noted that as
- *   flow is add to the indexed list, memory of the indexed
- *   list points to maybe changed as flow destroyed.
  * @param[in] flow_idx
  *   Index of flow to destroy.
  */
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW], flow_idx);
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
 
 	if (!flow)
 		return;
+	MLX5_ASSERT(flow->type == type);
 	/*
 	 * Update RX queue flags only if port is started, otherwise it is
 	 * already clean.
@@ -6570,12 +6532,6 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
 	flow_drv_destroy(dev, flow);
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
-			     flow_idx, flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
 	if (flow->tunnel) {
 		struct mlx5_flow_tunnel *tunnel;
 
@@ -6585,7 +6541,7 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
 	flow_mreg_del_copy_action(dev, flow);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
+	mlx5_ipool_free(priv->flows[type], flow_idx);
 }
 
 /**
@@ -6593,18 +6549,21 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list.
+ * @param type
+ *   Flow type to be flushed.
  * @param active
  *   If flushing is called avtively.
  */
 void
-mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
+mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+		     bool active)
 {
-	uint32_t num_flushed = 0;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t num_flushed = 0, fidx = 1;
+	struct rte_flow *flow;
 
-	while (*list) {
-		flow_list_destroy(dev, list, *list);
+	MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
+		flow_list_destroy(dev, type, fidx);
 		num_flushed++;
 	}
 	if (active) {
@@ -6776,18 +6735,19 @@ mlx5_flow_pop_thread_workspace(void)
  * @return the number of flows not released.
  */
 int
-mlx5_flow_verify(struct rte_eth_dev *dev)
+mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
-	uint32_t idx;
-	int ret = 0;
+	uint32_t idx = 0;
+	int ret = 0, i;
 
-	ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
-		      flow, next) {
-		DRV_LOG(DEBUG, "port %u flow %p still referenced",
-			dev->data->port_id, (void *)flow);
-		++ret;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
+			DRV_LOG(DEBUG, "port %u flow %p still referenced",
+				dev->data->port_id, (void *)flow);
+			ret++;
+		}
 	}
 	return ret;
 }
@@ -6807,7 +6767,6 @@ int
 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 			    uint32_t queue)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.egress = 1,
 		.priority = 0,
@@ -6840,8 +6799,8 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 	actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
 	actions[0].conf = &jump;
 	actions[1].type = RTE_FLOW_ACTION_TYPE_END;
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx) {
 		DRV_LOG(DEBUG,
 			"Failed to create ctrl flow: rte_errno(%d),"
@@ -6930,8 +6889,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 		action_rss.types = 0;
 	for (i = 0; i != priv->reta_idx_n; ++i)
 		queue[i] = (*priv->reta_idx)[i];
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx)
 		return -rte_errno;
 	return 0;
@@ -6972,7 +6931,6 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev,
 int
 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	/*
 	 * The LACP matching is done by only using ether type since using
 	 * a multicast dst mac causes kernel to give low priority to this flow.
@@ -7006,8 +6964,9 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 		},
 	};
 	struct rte_flow_error error;
-	uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+					&attr, items, actions,
+					false, &error);
 
 	if (!flow_idx)
 		return -rte_errno;
@@ -7025,9 +6984,8 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
 		  struct rte_flow *flow,
 		  struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+				(uintptr_t)(void *)flow);
 	return 0;
 }
 
@@ -7041,9 +6999,7 @@ int
 mlx5_flow_flush(struct rte_eth_dev *dev,
 		struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
 	return 0;
 }
 
@@ -7094,8 +7050,7 @@ flow_drv_query(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct mlx5_flow_driver_ops *fops;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW],
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 					       flow_idx);
 	enum mlx5_flow_drv_type ftype;
 
@@ -7961,14 +7916,14 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		if (!config->dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
-		flow_idx = flow_list_create(dev, NULL, &attr, items,
-					    actions, false, &error);
-		flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
+					items, actions, false, &error);
+		flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 				      flow_idx);
 		if (!flow)
 			continue;
 		config->flow_mreg_c[n++] = idx;
-		flow_list_destroy(dev, NULL, flow_idx);
+		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
 	}
 	for (; n < MLX5_MREG_C_NUM; ++n)
 		config->flow_mreg_c[n] = REG_NON;
@@ -8163,8 +8118,7 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
 	/* dump all */
 	if (!flow_idx) {
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-		ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
-						priv->flows, idx, flow, next)
+		MLX5_IPOOL_FOREACH(priv->flows[MLX5_FLOW_TYPE_GEN], idx, flow)
 			mlx5_flow_dev_dump_ipool(dev, flow, file, error);
 #endif
 		return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
@@ -8172,8 +8126,8 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
 					sh->tx_domain, file);
 	}
 	/* dump one */
-	flow = mlx5_ipool_get(priv->sh->ipool
-			[MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
+	flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
+			(uintptr_t)(void *)flow_idx);
 	if (!flow)
 		return -ENOENT;
 
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 7d97c5880f..adadcfa425 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -998,9 +998,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 2f4c0eeb5b..49bc85f7b3 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13901,6 +13901,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ae7fcca229..7cb8920d6b 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index e30b682822..17716b66c9 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
-- 
2.25.1

[dpdk-dev] [PATCH v5 06/26] net/mlx5: optimize modify header action memory

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Define the types of the modify header action fields to be with the
minimum size needed for the optional values range.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_glue.h b/drivers/common/mlx5/linux/mlx5_glue.h
index 61f40d5478..f39ef2dac7 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.h
+++ b/drivers/common/mlx5/linux/mlx5_glue.h
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
diff --git a/drivers/net/mlx5/linux/mlx5_flow_os.h b/drivers/net/mlx5/linux/mlx5_flow_os.h
index cee685015b..1926d26410 100644
--- a/drivers/net/mlx5/linux/mlx5_flow_os.h
+++ b/drivers/net/mlx5/linux/mlx5_flow_os.h
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index adadcfa425..f1ebc153f2 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -524,11 +524,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 49bc85f7b3..0f8b916991 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5006,21 +5006,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5622,10 +5622,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
-- 
2.25.1

[dpdk-dev] [PATCH v5 07/26] net/mlx5: remove cache term from the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The internal mlx5 list tool is used mainly when the list objects need to
be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread
cache management.

To prevent confusing, remove the current "cache" term from the API's
names.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 561 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 456 insertions(+), 461 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 867f450cab..5d537fa3e4 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -509,8 +509,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1713,7 +1713,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1774,7 +1774,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index b5c05b12e8..5cd2cfb07c 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1612,7 +1612,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index facab913bf..de5cd2e06e 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1137,10 +1137,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1244,7 +1244,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1382,7 +1382,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1392,7 +1392,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index f1ebc153f2..cbae4f1872 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -468,7 +468,7 @@ struct mlx5_flow_dv_match_params {
 
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
-	struct mlx5_cache_entry entry; /**< Pointer to the next element. */
+	struct mlx5_list_entry entry; /**< Pointer to the next element. */
 	struct mlx5_flow_tbl_resource *tbl;
 	/**< Pointer to the table(group) the matcher associated with. */
 	void *matcher_object; /**< Pointer to DV matcher */
@@ -548,7 +548,7 @@ struct mlx5_flow_dv_jump_tbl_resource {
 
 /* Port ID resource structure. */
 struct mlx5_flow_dv_port_id_action_resource {
-	struct mlx5_cache_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Action object. */
 	uint32_t port_id; /**< Port ID value. */
 	uint32_t idx; /**< Indexed pool memory index. */
@@ -556,7 +556,7 @@ struct mlx5_flow_dv_port_id_action_resource {
 
 /* Push VLAN action resource structure */
 struct mlx5_flow_dv_push_vlan_action_resource {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* Cache entry. */
 	void *action; /**< Action object. */
 	uint8_t ft_type; /**< Flow table type, Rx, Tx or FDB. */
 	rte_be32_t vlan_tag; /**< VLAN tag value. */
@@ -591,7 +591,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_cache_list matchers;
+	struct mlx5_list matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
@@ -632,7 +632,7 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	union {
 		void *verbs_action; /**< Verbs sample action object. */
 		void **sub_actions; /**< Sample sub-action array. */
@@ -654,7 +654,7 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
@@ -1631,43 +1631,45 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
 
-int flow_dv_matcher_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *ctx);
-struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_port_id_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_sample_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
-			      struct mlx5_cache_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
-				struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_dest_array_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry);
+int flow_dv_matcher_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *ctx);
+void flow_dv_matcher_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
+						 struct mlx5_list_entry *entry,
+						 void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 0f8b916991..3aa3192dd7 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3607,18 +3607,17 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 			     uint64_t key __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
-
-	cache_resource = container_of(entry,
-				      struct mlx5_flow_dv_encap_decap_resource,
-				      entry);
-	if (resource->reformat_type == cache_resource->reformat_type &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->flags == cache_resource->flags &&
-	    resource->size == cache_resource->size &&
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
+
+	resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
+				entry);
+	if (resource->reformat_type == ctx_resource->reformat_type &&
+	    resource->ft_type == ctx_resource->ft_type &&
+	    resource->flags == ctx_resource->flags &&
+	    resource->size == ctx_resource->size &&
 	    !memcmp((const void *)resource->buf,
-		    (const void *)cache_resource->buf,
+		    (const void *)ctx_resource->buf,
 		    resource->size))
 		return 0;
 	return -1;
@@ -3645,31 +3644,30 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 	uint32_t idx;
 	int ret;
 
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
 	/* Register new encap/decap resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-				       &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
-	cache_resource->idx = idx;
-	ret = mlx5_flow_os_create_flow_action_packet_reformat
-					(sh->ctx, domain, cache_resource,
-					 &cache_resource->action);
+	*resource = *ctx_resource;
+	resource->idx = idx;
+	ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
+							      resource,
+							     &resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3678,7 +3676,7 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 		return NULL;
 	}
 
-	return &cache_resource->entry;
+	return &resource->entry;
 }
 
 /**
@@ -3782,8 +3780,8 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
@@ -3793,30 +3791,30 @@ flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->port_id != res->port_id;
 }
 
-struct mlx5_cache_entry *
-flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_port_id_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate port_id action cache memory");
+				   "cannot allocate port_id action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
 							ref->port_id,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3824,8 +3822,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 				   "cannot create action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3833,8 +3831,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3846,30 +3844,30 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_port_id_action_resource_register
 			(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_port_id_action_resource *resource,
+			 struct mlx5_flow_dv_port_id_action_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->dv.port_id_action = cache;
-	dev_flow->handle->rix_port_id_action = cache->idx;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->dv.port_id_action = resource;
+	dev_flow->handle->rix_port_id_action = resource->idx;
 	return 0;
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
@@ -3879,28 +3877,28 @@ flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
-struct mlx5_cache_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	struct mlx5dv_dr_domain *domain;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate push_vlan action cache memory");
+				   "cannot allocate push_vlan action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
@@ -3908,7 +3906,7 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 	else
 		domain = sh->tx_domain;
 	ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3916,8 +3914,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 				   "cannot create push vlan action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3925,8 +3923,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
  *
  * @param [in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3938,25 +3936,25 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_push_vlan_action_resource_register
 		       (struct rte_eth_dev *dev,
-			struct mlx5_flow_dv_push_vlan_action_resource *resource,
+			struct mlx5_flow_dv_push_vlan_action_resource *ref,
 			struct mlx5_flow *dev_flow,
 			struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
+	resource = container_of(entry, typeof(*resource), entry);
 
-	dev_flow->handle->dvh.rix_push_vlan = cache->idx;
-	dev_flow->dv.push_vlan_res = cache;
+	dev_flow->handle->dvh.rix_push_vlan = resource->idx;
+	dev_flow->dv.push_vlan_res = resource;
 	return 0;
 }
 
@@ -9957,13 +9955,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 			return NULL;
 		}
 	}
-	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache",
+	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh,
-			     flow_dv_matcher_create_cb,
-			     flow_dv_matcher_match_cb,
-			     flow_dv_matcher_remove_cb);
+	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+			 flow_dv_matcher_create_cb,
+			 flow_dv_matcher_match_cb,
+			 flow_dv_matcher_remove_cb);
 	return &tbl_data->entry;
 }
 
@@ -10091,7 +10089,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_cache_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(&tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10119,8 +10117,8 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
@@ -10133,15 +10131,15 @@ flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
 		      (const void *)ref->mask.buf, ref->mask.size);
 }
 
-struct mlx5_cache_entry *
-flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_matcher_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5dv_flow_matcher_attr dv_attr = {
 		.type = IBV_FLOW_ATTR_NORMAL,
 		.match_mask = (void *)&ref->mask,
@@ -10150,29 +10148,30 @@ flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
 							    typeof(*tbl), tbl);
 	int ret;
 
-	cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY);
-	if (!cache) {
+	resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
+			       SOCKET_ID_ANY);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	dv_attr.match_criteria_enable =
-		flow_dv_matcher_enable(cache->mask.buf);
+		flow_dv_matcher_enable(resource->mask.buf);
 	dv_attr.priority = ref->priority;
 	if (tbl->is_egress)
 		dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
 	ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
-					       &cache->matcher_object);
+					       &resource->matcher_object);
 	if (ret) {
-		mlx5_free(cache);
+		mlx5_free(resource);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	return &cache->entry;
+	return &resource->entry;
 }
 
 /**
@@ -10201,8 +10200,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 			 uint32_t group_id,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5_flow_tbl_resource *tbl;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 	struct mlx5_flow_cb_ctx ctx = {
@@ -10222,15 +10221,15 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 					  "cannot allocate ref memory");
 	}
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->handle->dvh.matcher = cache;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.matcher = resource;
 	return 0;
 }
 
@@ -10298,15 +10297,15 @@ flow_dv_tag_resource_register
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_tag_resource *cache_resource;
+	struct mlx5_flow_dv_tag_resource *resource;
 	struct mlx5_hlist_entry *entry;
 
 	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
 	if (entry) {
-		cache_resource = container_of
-			(entry, struct mlx5_flow_dv_tag_resource, entry);
-		dev_flow->handle->dvh.rix_tag = cache_resource->idx;
-		dev_flow->dv.tag_resource = cache_resource;
+		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
+					entry);
+		dev_flow->handle->dvh.rix_tag = resource->idx;
+		dev_flow->dv.tag_resource = resource;
 		return 0;
 	}
 	return -rte_errno;
@@ -10633,68 +10632,69 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused,
-			struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-
-	if (resource->ratio == cache_resource->ratio &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->ft_id == cache_resource->ft_id &&
-	    resource->set_action == cache_resource->set_action &&
-	    !memcmp((void *)&resource->sample_act,
-		    (void *)&cache_resource->sample_act,
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+
+	if (ctx_resource->ratio == resource->ratio &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    ctx_resource->ft_id == resource->ft_id &&
+	    ctx_resource->set_action == resource->set_action &&
+	    !memcmp((void *)&ctx_resource->sample_act,
+		    (void *)&resource->sample_act,
 		    sizeof(struct mlx5_flow_sub_actions_list))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
 		flow_dv_sample_sub_actions_release(dev,
-						&resource->sample_idx);
+						   &ctx_resource->sample_idx);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	void **sample_dv_actions = resource->sub_actions;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	void **sample_dv_actions = ctx_resource->sub_actions;
+	struct mlx5_flow_dv_sample_resource *resource;
 	struct mlx5dv_dr_flow_sampler_attr sampler_attr;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	struct mlx5_flow_tbl_resource *tbl;
 	uint32_t idx = 0;
 	const uint32_t next_ft_step = 1;
-	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
+	uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
 	uint8_t is_egress = 0;
 	uint8_t is_transfer = 0;
 	struct rte_flow_error *error = ctx->error;
 
 	/* Register new sample resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	/* Create normal path table level */
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		is_transfer = 1;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
 		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
 					is_egress, is_transfer,
@@ -10707,8 +10707,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 					  "for sample");
 		goto error;
 	}
-	cache_resource->normal_path_tbl = tbl;
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
+	resource->normal_path_tbl = tbl;
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
 		if (!sh->default_miss_action) {
 			rte_flow_error_set(error, ENOMEM,
 						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10717,33 +10717,33 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 						"created");
 			goto error;
 		}
-		sample_dv_actions[resource->sample_act.actions_num++] =
+		sample_dv_actions[ctx_resource->sample_act.actions_num++] =
 						sh->default_miss_action;
 	}
 	/* Create a DR sample action */
-	sampler_attr.sample_ratio = cache_resource->ratio;
+	sampler_attr.sample_ratio = resource->ratio;
 	sampler_attr.default_next_table = tbl->obj;
-	sampler_attr.num_sample_actions = resource->sample_act.actions_num;
+	sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
 	sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
 							&sample_dv_actions[0];
-	sampler_attr.action = cache_resource->set_action;
+	sampler_attr.action = resource->set_action;
 	if (mlx5_os_flow_dr_create_flow_action_sampler
-			(&sampler_attr, &cache_resource->verbs_action)) {
+			(&sampler_attr, &resource->verbs_action)) {
 		rte_flow_error_set(error, ENOMEM,
 					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					NULL, "cannot create sample action");
 		goto error;
 	}
-	cache_resource->idx = idx;
-	cache_resource->dev = dev;
-	return &cache_resource->entry;
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
 error:
-	if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
 		flow_dv_sample_sub_actions_release(dev,
-						   &cache_resource->sample_idx);
-	if (cache_resource->normal_path_tbl)
+						   &resource->sample_idx);
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-				cache_resource->normal_path_tbl);
+				resource->normal_path_tbl);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
 	return NULL;
 
@@ -10754,8 +10754,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to sample resource.
+ * @param[in] ref
+ *   Pointer to sample resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10766,66 +10766,66 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_sample_resource *resource,
+			 struct mlx5_flow_dv_sample_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_sample = cache_resource->idx;
-	dev_flow->dv.sample_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_sample = resource->idx;
+	dev_flow->dv.sample_res = resource;
 	return 0;
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused,
-			    struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
-	if (resource->num_of_dest == cache_resource->num_of_dest &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    !memcmp((void *)cache_resource->sample_act,
-		    (void *)resource->sample_act,
-		   (resource->num_of_dest *
+	if (ctx_resource->num_of_dest == resource->num_of_dest &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    !memcmp((void *)resource->sample_act,
+		    (void *)ctx_resource->sample_act,
+		   (ctx_resource->num_of_dest *
 		   sizeof(struct mlx5_flow_sub_actions_list)))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
-		for (idx = 0; idx < resource->num_of_dest; idx++)
+		for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 			flow_dv_sample_sub_actions_release(dev,
-					&resource->sample_idx[idx]);
+					&ctx_resource->sample_idx[idx]);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
 	struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -10838,23 +10838,23 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	int ret;
 
 	/* Register new destination array resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
 					    &res_idx);
-	if (!cache_resource) {
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
 				 mlx5_malloc(MLX5_MEM_ZERO,
 				 sizeof(struct mlx5dv_dr_action_dest_attr),
@@ -10867,7 +10867,7 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 			goto error;
 		}
 		dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
-		sample_act = &resource->sample_act[idx];
+		sample_act = &ctx_resource->sample_act[idx];
 		action_flags = sample_act->action_flags;
 		switch (action_flags) {
 		case MLX5_FLOW_ACTION_QUEUE:
@@ -10898,9 +10898,9 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	/* create a dest array actioin */
 	ret = mlx5_os_flow_dr_create_flow_action_dest_array
 						(domain,
-						 cache_resource->num_of_dest,
+						 resource->num_of_dest,
 						 dest_attr,
-						 &cache_resource->action);
+						 &resource->action);
 	if (ret) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10908,19 +10908,18 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 				   "cannot create destination array action");
 		goto error;
 	}
-	cache_resource->idx = res_idx;
-	cache_resource->dev = dev;
-	for (idx = 0; idx < resource->num_of_dest; idx++)
+	resource->idx = res_idx;
+	resource->dev = dev;
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 		mlx5_free(dest_attr[idx]);
-	return &cache_resource->entry;
+	return &resource->entry;
 error:
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[idx]);
+						   &resource->sample_idx[idx]);
 		if (dest_attr[idx])
 			mlx5_free(dest_attr[idx]);
 	}
-
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
 	return NULL;
 }
@@ -10930,8 +10929,8 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to destination array resource.
+ * @param[in] ref
+ *   Pointer to destination array resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10942,25 +10941,25 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_dest_array_resource *resource,
+			 struct mlx5_flow_dv_dest_array_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_dest_array = cache_resource->idx;
-	dev_flow->dv.dest_array_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_dest_array = resource->idx;
+	dev_flow->dv.dest_array_res = resource;
 	return 0;
 }
 
@@ -13402,14 +13401,15 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			  struct mlx5_cache_entry *entry)
+flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+			  struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache),
-							  entry);
+	struct mlx5_flow_dv_matcher *resource = container_of(entry,
+							     typeof(*resource),
+							     entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object));
-	mlx5_free(cache);
+	claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
+	mlx5_free(resource);
 }
 
 /**
@@ -13433,7 +13433,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13452,7 +13452,7 @@ flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
-		container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
@@ -13474,15 +13474,14 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
 				     uint32_t encap_decap_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-					encap_decap_idx);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+				  encap_decap_idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->action);
-	return mlx5_hlist_unregister(priv->sh->encaps_decaps,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
 }
 
 /**
@@ -13544,15 +13543,15 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry)
+flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_port_id_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+				  container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
 
 /**
@@ -13571,14 +13570,14 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 					uint32_t port_id)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->port_id_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13601,15 +13600,15 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry)
+flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
 
 /**
@@ -13628,15 +13627,15 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 					  struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx = handle->dvh.rix_push_vlan;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->push_vlan_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13673,26 +13672,24 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry)
+flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	if (cache_resource->verbs_action)
+	if (resource->verbs_action)
 		claim_zero(mlx5_flow_os_destroy_flow_action
-				(cache_resource->verbs_action));
-	if (cache_resource->normal_path_tbl)
+						      (resource->verbs_action));
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-			cache_resource->normal_path_tbl);
-	flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			cache_resource->idx);
-	DRV_LOG(DEBUG, "sample resource %p: removed",
-		(void *)cache_resource);
+					     resource->normal_path_tbl);
+	flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
+	DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
 }
 
 /**
@@ -13711,38 +13708,36 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 				     struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			 handle->dvh.rix_sample);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+				  handle->dvh.rix_sample);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->verbs_action);
-	return mlx5_cache_unregister(&priv->sh->sample_action_list,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->verbs_action);
+	return mlx5_list_unregister(&priv->sh->sample_action_list,
+				    &resource->entry);
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	uint32_t i = 0;
 
-	MLX5_ASSERT(cache_resource->action);
-	if (cache_resource->action)
-		claim_zero(mlx5_flow_os_destroy_flow_action
-					(cache_resource->action));
-	for (; i < cache_resource->num_of_dest; i++)
+	MLX5_ASSERT(resource->action);
+	if (resource->action)
+		claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	for (; i < resource->num_of_dest; i++)
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[i]);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			cache_resource->idx);
+						   &resource->sample_idx[i]);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
 	DRV_LOG(DEBUG, "destination array resource %p: removed",
-		(void *)cache_resource);
+		(void *)resource);
 }
 
 /**
@@ -13761,15 +13756,15 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 				    struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_dest_array_resource *cache;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			       handle->dvh.rix_dest_array);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				  handle->dvh.rix_dest_array);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->dest_array_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->dest_array_list,
+				    &resource->entry);
 }
 
 static void
@@ -14619,7 +14614,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 			claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
 			tbl = container_of(color_rule->matcher->tbl,
 					typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 						&color_rule->matcher->entry);
 			TAILQ_REMOVE(&sub_policy->color_rules[i],
 					color_rule, next_port);
@@ -15412,8 +15407,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_cache_unregister(&tbl->matchers,
-				      &mtrmng->def_matcher[i]->entry);
+			mlx5_list_unregister(&tbl->matchers,
+					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
 		for (j = 0; j < MLX5_REG_BITS; j++) {
@@ -15422,8 +15417,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_cache_unregister(&tbl->matchers,
-					&mtrmng->drop_matcher[i][j]->entry);
+				mlx5_list_unregister(&tbl->matchers,
+					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
 		}
@@ -15521,7 +15516,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 			struct mlx5_flow_dv_matcher **policy_matcher,
 			struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
@@ -15557,7 +15552,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -15665,7 +15660,7 @@ __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
 			struct mlx5_flow_tbl_data_entry *tbl =
 				container_of(color_rule->matcher->tbl,
 						typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 						&color_rule->matcher->entry);
 		}
 		mlx5_free(color_rule);
@@ -16017,7 +16012,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 						     0, &error);
 	uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
 	uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
 			.size = sizeof(matcher.mask.buf) -
@@ -16063,7 +16058,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -16100,7 +16095,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
@@ -16517,7 +16512,7 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
 			struct mlx5_flow_tbl_data_entry *tbl =
 				container_of(color_rule->matcher->tbl,
 						typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 						&color_rule->matcher->entry);
 		}
 		mlx5_free(color_rule);
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 1b264e5994..3dcc71d51d 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index bb9a908087..8395332507 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index f9557c09ff..4536ca807d 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,29 +9,29 @@
 #include "mlx5_utils.h"
 
 
-/********************* Cache list ************************/
+/********************* MLX5 list ************************/
 
-static struct mlx5_cache_entry *
-mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry __rte_unused,
+static struct mlx5_list_entry *
+mlx5_list_default_create_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry __rte_unused,
 			     void *ctx __rte_unused)
 {
 	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
 }
 
 static void
-mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
 }
 
 int
-mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
+mlx5_list_create(struct mlx5_list *list, const char *name,
 		     uint32_t entry_size, void *ctx,
-		     mlx5_cache_create_cb cb_create,
-		     mlx5_cache_match_cb cb_match,
-		     mlx5_cache_remove_cb cb_remove)
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove)
 {
 	MLX5_ASSERT(list);
 	if (!cb_match || (!cb_create ^ !cb_remove))
@@ -40,19 +40,19 @@ mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
+	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
+	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
 	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	LIST_INIT(&list->head);
 	return 0;
 }
 
-static struct mlx5_cache_entry *
-__cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	LIST_FOREACH(entry, &list->head, next) {
 		if (list->cb_match(list, entry, ctx))
@@ -60,7 +60,7 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 		if (reuse) {
 			__atomic_add_fetch(&entry->ref_cnt, 1,
 					   __ATOMIC_RELAXED);
-			DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
 		break;
@@ -68,33 +68,33 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_cache_entry *
-cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __cache_lookup(list, ctx, reuse);
+	entry = __list_lookup(list, ctx, reuse);
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	return cache_lookup(list, ctx, false);
+	return list_lookup(list, ctx, false);
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	uint32_t prev_gen_cnt = 0;
 
 	MLX5_ASSERT(list);
 	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
 	/* Lookup with read lock, reuse if found. */
-	entry = cache_lookup(list, ctx, true);
+	entry = list_lookup(list, ctx, true);
 	if (entry)
 		return entry;
 	/* Not found, append with write lock - block read from other threads. */
@@ -102,13 +102,13 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __cache_lookup(list, ctx, true);
+		entry = __list_lookup(list, ctx, true);
 		if (entry)
 			goto done;
 	}
 	entry = list->cb_create(list, entry, ctx);
 	if (!entry) {
-		DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
+		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
 			list->name, (void *)entry);
 		goto done;
 	}
@@ -116,7 +116,7 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	LIST_INSERT_HEAD(&list->head, entry, next);
 	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 done:
 	rte_rwlock_write_unlock(&list->lock);
@@ -124,12 +124,12 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 }
 
 int
-mlx5_cache_unregister(struct mlx5_cache_list *list,
-		      struct mlx5_cache_entry *entry)
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
 {
 	rte_rwlock_write_lock(&list->lock);
 	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 	if (--entry->ref_cnt) {
 		rte_rwlock_write_unlock(&list->lock);
@@ -140,15 +140,15 @@ mlx5_cache_unregister(struct mlx5_cache_list *list,
 	LIST_REMOVE(entry, next);
 	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 		list->name, (void *)entry);
 	return 0;
 }
 
 void
-mlx5_cache_list_destroy(struct mlx5_cache_list *list)
+mlx5_list_destroy(struct mlx5_list *list)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	MLX5_ASSERT(list);
 	/* no LIST_FOREACH_SAFE, using while instead */
@@ -156,14 +156,14 @@ mlx5_cache_list_destroy(struct mlx5_cache_list *list)
 		entry = LIST_FIRST(&list->head);
 		LIST_REMOVE(entry, next);
 		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
 			list->name, (void *)entry);
 	}
 	memset(list, 0, sizeof(*list));
 }
 
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
+mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
 	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index a509b0a4eb..cfb3cb6180 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -297,19 +297,19 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ cache list *****************************/
+/************************ mlx5 list *****************************/
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
 
-struct mlx5_cache_list;
+struct mlx5_list;
 
 /**
- * Structure of the entry in the cache list, user should define its own struct
+ * Structure of the entry in the mlx5 list, user should define its own struct
  * that contains this in order to store the data.
  */
-struct mlx5_cache_entry {
-	LIST_ENTRY(mlx5_cache_entry) next; /* Entry pointers in the list. */
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* Reference count. */
 };
 
@@ -317,18 +317,18 @@ struct mlx5_cache_entry {
  * Type of callback function for entry removal.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
-				     struct mlx5_cache_entry *entry);
+typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
+				     struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -337,14 +337,14 @@ typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
-				   struct mlx5_cache_entry *entry, void *ctx);
+typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+				   struct mlx5_list_entry *entry, void *ctx);
 
 /**
- * Type of function for user defined cache list entry creation.
+ * Type of function for user defined mlx5 list entry creation.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The new allocated entry, NULL if list entry size unspecified,
  *   New entry has to be allocated in callback and return.
@@ -354,46 +354,46 @@ typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_cache_entry *(*mlx5_cache_create_cb)
-				 (struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry,
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry,
 				  void *ctx);
 
 /**
- * Linked cache list structure.
+ * Linked mlx5 list structure.
  *
- * Entry in cache list could be reused if entry already exists,
+ * Entry in mlx5 list could be reused if entry already exists,
  * reference count will increase and the existing entry returns.
  *
  * When destroy an entry from list, decrease reference count and only
  * destroy when no further reference.
  *
- * Linked list cache is designed for limited number of entries cache,
+ * Linked list is designed for limited number of entries,
  * read mostly, less modification.
  *
- * For huge amount of entries cache, please consider hash list cache.
+ * For huge amount of entries, please consider hash list.
  *
  */
-struct mlx5_cache_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the cache list. */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	mlx5_cache_create_cb cb_create; /**< entry create callback. */
-	mlx5_cache_match_cb cb_match; /**< entry match callback. */
-	mlx5_cache_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_cache_head, mlx5_cache_entry) head;
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
 };
 
 /**
- * Initialize a cache list.
+ * Create a mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
  * @param name
- *   Name of the cache list.
+ *   Name of the mlx5 list.
  * @param entry_size
  *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
@@ -407,11 +407,11 @@ struct mlx5_cache_list {
  * @return
  *   0 on success, otherwise failure.
  */
-int mlx5_cache_list_init(struct mlx5_cache_list *list,
+int mlx5_list_create(struct mlx5_list *list,
 			 const char *name, uint32_t entry_size, void *ctx,
-			 mlx5_cache_create_cb cb_create,
-			 mlx5_cache_match_cb cb_match,
-			 mlx5_cache_remove_cb cb_remove);
+			 mlx5_list_create_cb cb_create,
+			 mlx5_list_match_cb cb_match,
+			 mlx5_list_remove_cb cb_remove);
 
 /**
  * Search an entry matching the key.
@@ -420,18 +420,18 @@ int mlx5_cache_list_init(struct mlx5_cache_list *list,
  * this function only in main thread.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  * @param ctx
  *   Common context parameter used by entry callback function.
  *
  * @return
- *   Pointer of the cache entry if found, NULL otherwise.
+ *   Pointer of the list entry if found, NULL otherwise.
  */
-struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
 					   void *ctx);
 
 /**
- * Reuse or create an entry to the cache list.
+ * Reuse or create an entry to the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
@@ -441,42 +441,42 @@ struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
  * @return
  *   registered entry on success, NULL otherwise
  */
-struct mlx5_cache_entry *mlx5_cache_register(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
 					     void *ctx);
 
 /**
- * Remove an entry from the cache list.
+ * Remove an entry from the mlx5 list.
  *
  * User should guarantee the validity of the entry.
  *
  * @param list
  *   Pointer to the hast list.
  * @param entry
- *   Entry to be removed from the cache list table.
+ *   Entry to be removed from the mlx5 list table.
  * @return
  *   0 on entry removed, 1 on entry still referenced.
  */
-int mlx5_cache_unregister(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry);
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
 
 /**
- * Destroy the cache list.
+ * Destroy the mlx5 list.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  */
-void mlx5_cache_list_destroy(struct mlx5_cache_list *list);
+void mlx5_list_destroy(struct mlx5_list *list);
 
 /**
- * Get entry number from the cache list.
+ * Get entry number from the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list.
  * @return
- *   Cache list entry number.
+ *   mlx5 list entry number.
  */
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list);
+mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /********************************* indexed pool *************************/
 
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 17716b66c9..bcf72dc6db 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,10 +610,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v5 08/26] net/mlx5: add per lcore cache to the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When mlx5 list object is accessed by multiple cores, the list lock
counter is all the time written by all the cores what increases cache
misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup
operation, all the other threads coming to do an operation in the list
are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when
the list objects are mostly reused.

Synchronization with atomic operations should be done in order to
allow threads to unregister an entry from other thread cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 5d537fa3e4..25e4bdb7a6 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1713,10 +1721,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index de5cd2e06e..1141e12977 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1388,6 +1388,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index cbae4f1872..2016adc6ee 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1646,7 +1646,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1654,6 +1658,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1662,6 +1671,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1670,6 +1684,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 3aa3192dd7..2d0e82fab7 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3826,6 +3826,39 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_port_id_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate port_id action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
+}
+
 /**
  * Find existing table port ID resource or create and register a new one.
  *
@@ -3918,6 +3951,39 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate push_vlan action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
+}
+
 /**
  * Find existing push vlan resource or create and register a new one.
  *
@@ -9892,6 +9958,36 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 	return match_criteria_enable;
 }
 
+static struct mlx5_list_entry *
+flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_matcher *ref = ctx->data;
+	struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
+							    typeof(*tbl), tbl);
+	struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
+							    sizeof(*resource),
+							    0, SOCKET_ID_ANY);
+
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot create matcher");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->tbl = &tbl->tbl;
+	return &resource->entry;
+}
+
+static void
+flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 struct mlx5_hlist_entry *
 flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 {
@@ -9958,10 +10054,12 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
 			 flow_dv_matcher_create_cb,
 			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb);
+			 flow_dv_matcher_remove_cb,
+			 flow_dv_matcher_clone_cb,
+			 flow_dv_matcher_clone_free_cb);
 	return &tbl_data->entry;
 }
 
@@ -10749,6 +10847,45 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 
 }
 
+struct mlx5_list_entry *
+flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t idx = 0;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_sample_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+			resource->idx);
+}
+
 /**
  * Find existing sample resource or create and register a new one.
  *
@@ -10924,6 +11061,46 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 	return NULL;
 }
 
+struct mlx5_list_entry *
+flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t res_idx = 0;
+	struct rte_flow_error *error = ctx->error;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				      &res_idx);
+	if (!resource) {
+		rte_flow_error_set(error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate dest-array memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = res_idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
+}
+
 /**
  * Find existing destination array resource or create and register a new one.
  *
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 3dcc71d51d..5450ddd388 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 8395332507..f8769da8dc 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1857,20 +1857,18 @@ mlx5_ind_table_obj_get(struct rte_eth_dev *dev, const uint16_t *queues,
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_ind_table_obj *ind_tbl;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		if ((ind_tbl->queues_n == queues_n) &&
 		    (memcmp(ind_tbl->queues, queues,
 			    ind_tbl->queues_n * sizeof(ind_tbl->queues[0]))
-		     == 0))
+		     == 0)) {
+			__atomic_fetch_add(&ind_tbl->refcnt, 1,
+					   __ATOMIC_RELAXED);
 			break;
+		}
 	}
-	if (ind_tbl) {
-		unsigned int i;
-
-		__atomic_fetch_add(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
-		for (i = 0; i != ind_tbl->queues_n; ++i)
-			mlx5_rxq_get(dev, ind_tbl->queues[i]);
-	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ind_tbl;
 }
 
@@ -1893,19 +1891,20 @@ mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
 			   bool standalone)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	unsigned int i;
+	unsigned int i, ret;
 
-	if (__atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED) == 0)
-		priv->obj_ops.ind_table_destroy(ind_tbl);
+	rte_rwlock_write_lock(&priv->ind_tbls_lock);
+	ret = __atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
+	if (!ret && !standalone)
+		LIST_REMOVE(ind_tbl, next);
+	rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	if (ret)
+		return 1;
+	priv->obj_ops.ind_table_destroy(ind_tbl);
 	for (i = 0; i != ind_tbl->queues_n; ++i)
 		claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
-	if (__atomic_load_n(&ind_tbl->refcnt, __ATOMIC_RELAXED) == 0) {
-		if (!standalone)
-			LIST_REMOVE(ind_tbl, next);
-		mlx5_free(ind_tbl);
-		return 0;
-	}
-	return 1;
+	mlx5_free(ind_tbl);
+	return 0;
 }
 
 /**
@@ -1924,12 +1923,14 @@ mlx5_ind_table_obj_verify(struct rte_eth_dev *dev)
 	struct mlx5_ind_table_obj *ind_tbl;
 	int ret = 0;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		DRV_LOG(DEBUG,
 			"port %u indirection table obj %p still referenced",
 			dev->data->port_id, (void *)ind_tbl);
 		++ret;
 	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ret;
 }
 
@@ -2015,8 +2016,11 @@ mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues,
 		mlx5_free(ind_tbl);
 		return NULL;
 	}
-	if (!standalone)
+	if (!standalone) {
+		rte_rwlock_write_lock(&priv->ind_tbls_lock);
 		LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
+		rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	}
 	return ind_tbl;
 }
 
@@ -2328,6 +2332,35 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 	return hrxq ? &hrxq->entry : NULL;
 }
 
+struct mlx5_list_entry *
+mlx5_hrxq_clone_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry,
+		    void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq;
+	uint32_t hrxq_idx = 0;
+
+	hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx);
+	if (!hrxq)
+		return NULL;
+	memcpy(hrxq, entry, sizeof(*hrxq) + MLX5_RSS_HASH_KEY_LEN);
+	hrxq->idx = hrxq_idx;
+	return &hrxq->entry;
+}
+
+void
+mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq->idx);
+}
+
 /**
  * Get an Rx Hash queue.
  *
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 4536ca807d..f505caed4e 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,57 +9,68 @@
 #include "mlx5_utils.h"
 
 
-/********************* MLX5 list ************************/
-
-static struct mlx5_list_entry *
-mlx5_list_default_create_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
-{
-	mlx5_free(entry);
-}
+/********************* mlx5 list ************************/
 
 int
-mlx5_list_create(struct mlx5_list *list, const char *name,
-		     uint32_t entry_size, void *ctx,
-		     mlx5_list_create_cb cb_create,
-		     mlx5_list_match_cb cb_match,
-		     mlx5_list_remove_cb cb_remove)
+mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	int i;
+
 	MLX5_ASSERT(list);
-	if (!cb_match || (!cb_create ^ !cb_remove))
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free)
 		return -1;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
+	list->cb_create = cb_create;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	LIST_INIT(&list->head);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry;
-
-	LIST_FOREACH(entry, &list->head, next) {
-		if (list->cb_match(list, entry, ctx))
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (list->cb_match(list, entry, ctx)) {
+			if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_ACQUIRE);
+				if (ret == 0) {
+					LIST_REMOVE(entry, next);
+					list->cb_clone_free(list, entry);
+				}
+			}
+			entry = nentry;
 			continue;
+		}
 		if (reuse) {
-			__atomic_add_fetch(&entry->ref_cnt, 1,
-					   __ATOMIC_RELAXED);
+			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+						 __ATOMIC_ACQUIRE);
+			if (ret == 1u) {
+				/* Entry was invalid before, free it. */
+				LIST_REMOVE(entry, next);
+				list->cb_clone_free(list, entry);
+				entry = nentry;
+				continue;
+			}
 			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
@@ -68,96 +79,141 @@ __list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_list_entry *
-list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry = NULL;
+	int i;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, ctx, reuse);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	return list_lookup(list, ctx, false);
+	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+
+	if (!lentry)
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
 }
 
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry, *lentry;
 	uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
-	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* Lookup in local cache. */
+	lentry = __list_lookup(list, lcore_index, ctx, true);
+	if (lentry)
+		return lentry;
 	/* Lookup with read lock, reuse if found. */
-	entry = list_lookup(list, ctx, true);
-	if (entry)
-		return entry;
+	rte_rwlock_read_lock(&list->lock);
+	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+	if (entry == NULL) {
+		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
+					       __ATOMIC_ACQUIRE);
+		rte_rwlock_read_unlock(&list->lock);
+	} else {
+		rte_rwlock_read_unlock(&list->lock);
+		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+	}
 	/* Not found, append with write lock - block read from other threads. */
 	rte_rwlock_write_lock(&list->lock);
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, ctx, true);
-		if (entry)
-			goto done;
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (entry) {
+			rte_rwlock_write_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
 	}
 	entry = list->cb_create(list, entry, ctx);
-	if (!entry) {
-		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
-			list->name, (void *)entry);
-		goto done;
+	if (entry) {
+		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+		if (!lentry) {
+			list->cb_remove(list, entry);
+		} else {
+			entry->ref_cnt = 1u;
+			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
+					 next);
+			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
+			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+				list->name, (void *)entry, entry->ref_cnt);
+		}
+
 	}
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(&list->head, entry, next);
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-done:
 	rte_rwlock_write_unlock(&list->lock);
-	return entry;
+	return lentry;
 }
 
 int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
+	struct mlx5_list_entry *gentry = entry->gentry;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
+	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
+		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		LIST_REMOVE(gentry, next);
+		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
-		return 1;
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
 	}
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-	__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	LIST_REMOVE(entry, next);
-	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-		list->name, (void *)entry);
-	return 0;
+	return 1;
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
+	int i;
 
 	MLX5_ASSERT(list);
-	/* no LIST_FOREACH_SAFE, using while instead */
-	while (!LIST_EMPTY(&list->head)) {
-		entry = LIST_FIRST(&list->head);
-		LIST_REMOVE(entry, next);
-		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
-			list->name, (void *)entry);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list, entry);
+			}
+		}
 	}
 	memset(list, 0, sizeof(*list));
 }
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index cfb3cb6180..9e3fe0cb85 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -310,9 +310,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -340,6 +345,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -376,15 +388,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -394,8 +408,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -408,10 +420,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index bcf72dc6db..8ced98f0dc 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,9 +610,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v5 09/26] net/mlx5: minimize list critical sections

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is
taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical
sections.

These operations are heavy and make the critical sections heavy because
they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation
counter in order to detect parallel allocations.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index f505caed4e..c4c9adb039 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 9e3fe0cb85..6dade8238d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -388,8 +388,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
-- 
2.25.1

[dpdk-dev] [PATCH v5 10/26] net/mlx5: manage list cache entries release

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When a cache entry is allocated by lcore A and is released by lcore B,
the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be
increased atomically when the non-original lcore decreases the reference
counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in
its cache, it will check the counter in order to free invalid entries in
its cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index c4c9adb039..13c7dbe1c2 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 6dade8238d..71da5ab4f9 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -311,11 +311,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v5 11/26] net/mlx5: relax the list utility atomic operations

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The atomic operation in the list utility no need a barriers because the
critical part are managed by RW lock.

Relax them.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 13c7dbe1c2..daecf37575 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
-- 
2.25.1

[dpdk-dev] [PATCH v5 12/26] net/mlx5: allocate list memory by the create API

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence
with the hlist utility.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  60 ++++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 139 insertions(+), 83 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 25e4bdb7a6..2a9a6c3bf8 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -456,6 +464,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -517,9 +541,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1721,11 +1759,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1784,7 +1824,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 5cd2cfb07c..d04b5d616f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1612,7 +1612,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 1141e12977..88b84d749a 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1137,10 +1137,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1382,7 +1382,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2016adc6ee..59a6f31d2e 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -591,7 +591,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 2d0e82fab7..5a536e3dff 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3889,7 +3889,7 @@ flow_dv_port_id_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -4014,7 +4014,7 @@ flow_dv_push_vlan_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10054,12 +10054,22 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
-			 flow_dv_matcher_create_cb,
-			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb,
-			 flow_dv_matcher_clone_cb,
-			 flow_dv_matcher_clone_free_cb);
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+					      flow_dv_matcher_create_cb,
+					      flow_dv_matcher_match_cb,
+					      flow_dv_matcher_remove_cb,
+					      flow_dv_matcher_clone_cb,
+					      flow_dv_matcher_clone_free_cb);
+	if (!tbl_data->matchers) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot create tbl matcher list");
+		mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
+		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
+		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
+		return NULL;
+	}
 	return &tbl_data->entry;
 }
 
@@ -10187,7 +10197,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10319,7 +10329,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
@@ -10916,7 +10926,7 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -11131,7 +11141,7 @@ flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -13610,7 +13620,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13753,7 +13763,7 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+	return mlx5_list_unregister(priv->sh->port_id_action_list,
 				    &resource->entry);
 }
 
@@ -13811,7 +13821,7 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+	return mlx5_list_unregister(priv->sh->push_vlan_action_list,
 				    &resource->entry);
 }
 
@@ -13892,7 +13902,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->verbs_action);
-	return mlx5_list_unregister(&priv->sh->sample_action_list,
+	return mlx5_list_unregister(priv->sh->sample_action_list,
 				    &resource->entry);
 }
 
@@ -13940,7 +13950,7 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->dest_array_list,
+	return mlx5_list_unregister(priv->sh->dest_array_list,
 				    &resource->entry);
 }
 
@@ -14791,7 +14801,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 			claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
 			tbl = container_of(color_rule->matcher->tbl,
 					typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 						&color_rule->matcher->entry);
 			TAILQ_REMOVE(&sub_policy->color_rules[i],
 					color_rule, next_port);
@@ -15584,7 +15594,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
@@ -15594,7 +15604,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_list_unregister(&tbl->matchers,
+				mlx5_list_unregister(tbl->matchers,
 					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
@@ -15729,7 +15739,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -15837,7 +15847,7 @@ __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
 			struct mlx5_flow_tbl_data_entry *tbl =
 				container_of(color_rule->matcher->tbl,
 						typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 						&color_rule->matcher->entry);
 		}
 		mlx5_free(color_rule);
@@ -16235,7 +16245,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -16272,7 +16282,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
@@ -16689,7 +16699,7 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
 			struct mlx5_flow_tbl_data_entry *tbl =
 				container_of(color_rule->matcher->tbl,
 						typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 						&color_rule->matcher->entry);
 		}
 		mlx5_free(color_rule);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index f8769da8dc..aa9e973d10 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index daecf37575..0be778935f 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 71da5ab4f9..ea64bb75c9 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -420,15 +420,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 8ced98f0dc..e6176e70d2 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v5 13/26] common/mlx5: move list utility to common

[Suanming Mou]

Hash list is planned to be implemented with the cache list code.

This commit moves the list utility to common directory.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 250 +++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h | 205 +++++++++++++++++++
 drivers/common/mlx5/version.map         |   7 +
 drivers/net/mlx5/mlx5_utils.c           | 251 ------------------------
 drivers/net/mlx5/mlx5_utils.h           | 197 -------------------
 6 files changed, 464 insertions(+), 448 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 306f2f1ab7..7fb7d40b38 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index ad2011e858..8bb8a6016d 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -11,6 +11,256 @@
 #include "mlx5_common_utils.h"
 #include "mlx5_common_log.h"
 
+/********************* mlx5 list ************************/
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+	int i;
+
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (name)
+		snprintf(list->name, sizeof(list->name), "%s", name);
+	list->ctx = ctx;
+	list->cb_create = cb_create;
+	list->cb_match = cb_match;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
+	rte_rwlock_init(&list->lock);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
+	return list;
+}
+
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+{
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_RELAXED) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_RELAXED);
+			}
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
+		}
+		entry = LIST_NEXT(entry, next);
+	}
+	return NULL;
+}
+
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry = NULL;
+	int i;
+
+	rte_rwlock_read_lock(&list->lock);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
+	rte_rwlock_read_unlock(&list->lock);
+	return entry;
+}
+
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
+{
+	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+
+	if (unlikely(!lentry))
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
+}
+
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	MLX5_ASSERT(list);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
+	rte_rwlock_read_lock(&list->lock);
+	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+	if (likely(entry)) {
+		rte_rwlock_read_unlock(&list->lock);
+		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+	}
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
+	rte_rwlock_write_lock(&list->lock);
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
+			rte_rwlock_write_unlock(&list->lock);
+			list->cb_remove(list, entry);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
+		}
+	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
+	rte_rwlock_write_unlock(&list->lock);
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
+	return local_entry;
+}
+
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	rte_rwlock_write_lock(&list->lock);
+	if (likely(gentry->ref_cnt == 0)) {
+		LIST_REMOVE(gentry, next);
+		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
+	}
+	rte_rwlock_write_unlock(&list->lock);
+	return 1;
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	struct mlx5_list_entry *entry;
+	int i;
+
+	MLX5_ASSERT(list);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list, entry);
+			}
+		}
+	}
+	mlx5_free(list);
+}
+
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list)
+{
+	MLX5_ASSERT(list);
+	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+}
+
 /********************* Hash List **********************/
 
 static struct mlx5_hlist_entry *
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index ed378ce9bd..96add6d003 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -7,6 +7,211 @@
 
 #include "mlx5_common.h"
 
+/************************ mlx5 list *****************************/
+
+/** Maximum size of string for naming. */
+#define MLX5_NAME_SIZE			32
+
+struct mlx5_list;
+
+/**
+ * Structure of the entry in the mlx5 list, user should define its own struct
+ * that contains this in order to store the data.
+ */
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
+	struct mlx5_list_entry *gentry;
+};
+
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
+} __rte_cache_aligned;
+
+/**
+ * Type of callback function for entry removal.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The entry in the list.
+ */
+typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
+				     struct mlx5_list_entry *entry);
+
+/**
+ * Type of function for user defined matching.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The entry in the list.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   0 if matching, non-zero number otherwise.
+ */
+typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+				   struct mlx5_list_entry *entry, void *ctx);
+
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
+/**
+ * Type of function for user defined mlx5 list entry creation.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The new allocated entry, NULL if list entry size unspecified,
+ *   New entry has to be allocated in callback and return.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   Pointer of entry on success, NULL otherwise.
+ */
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry,
+				  void *ctx);
+
+/**
+ * Linked mlx5 list structure.
+ *
+ * Entry in mlx5 list could be reused if entry already exists,
+ * reference count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ *
+ * Linked list is designed for limited number of entries,
+ * read mostly, less modification.
+ *
+ * For huge amount of entries, please consider hash list.
+ *
+ */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
+	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+};
+
+/**
+ * Create a mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param name
+ *   Name of the mlx5 list.
+ * @param ctx
+ *   Pointer to the list context data.
+ * @param cb_create
+ *   Callback function for entry create.
+ * @param cb_match
+ *   Callback function for entry match.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @return
+ *   List pointer on success, otherwise NULL.
+ */
+__rte_internal
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ * @param ctx
+ *   Common context parameter used by entry callback function.
+ *
+ * @return
+ *   Pointer of the list entry if found, NULL otherwise.
+ */
+__rte_internal
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
+					   void *ctx);
+
+/**
+ * Reuse or create an entry to the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param ctx
+ *   Common context parameter used by callback function.
+ *
+ * @return
+ *   registered entry on success, NULL otherwise
+ */
+__rte_internal
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
+					     void *ctx);
+
+/**
+ * Remove an entry from the mlx5 list.
+ *
+ * User should guarantee the validity of the entry.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @param entry
+ *   Entry to be removed from the mlx5 list table.
+ * @return
+ *   0 on entry removed, 1 on entry still referenced.
+ */
+__rte_internal
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
+
+/**
+ * Destroy the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ */
+__rte_internal
+void mlx5_list_destroy(struct mlx5_list *list);
+
+/**
+ * Get entry number from the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @return
+ *   mlx5 list entry number.
+ */
+__rte_internal
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list);
+
+/************************ Hash list *****************************/
+
 #define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
 #define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
 
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index b8be73a77b..e6586d6f6f 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -73,6 +73,13 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_list_create;
+	mlx5_list_register;
+	mlx5_list_unregister;
+	mlx5_list_lookup;
+	mlx5_list_get_entry_num;
+	mlx5_list_destroy;
+
 	mlx5_hlist_create;
 	mlx5_hlist_lookup;
 	mlx5_hlist_register;
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 0be778935f..e4e66ae4c5 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -8,257 +8,6 @@
 
 #include "mlx5_utils.h"
 
-
-/********************* mlx5 list ************************/
-
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
-{
-	struct mlx5_list *list;
-	int i;
-
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return NULL;
-	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
-	return list;
-}
-
-static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
-{
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
-	uint32_t ret;
-
-	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
-			if (reuse) {
-				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_RELAXED) - 1;
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
-					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
-				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_RELAXED);
-			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
-				return entry;
-			if (reuse && ret == 0)
-				entry->ref_cnt--; /* Invalid entry. */
-		}
-		entry = LIST_NEXT(entry, next);
-	}
-	return NULL;
-}
-
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry = NULL;
-	int i;
-
-	rte_rwlock_read_lock(&list->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
-		if (entry)
-			break;
-	}
-	rte_rwlock_read_unlock(&list->lock);
-	return entry;
-}
-
-static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
-		       struct mlx5_list_entry *gentry, void *ctx)
-{
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
-
-	if (unlikely(!lentry))
-		return NULL;
-	lentry->ref_cnt = 1u;
-	lentry->gentry = gentry;
-	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
-	return lentry;
-}
-
-static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
-{
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
-	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
-	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
-					       __ATOMIC_RELAXED);
-
-	while (inv_cnt != 0 && entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
-			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
-			inv_cnt--;
-		}
-		entry = nentry;
-	}
-}
-
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry, *local_entry;
-	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
-	MLX5_ASSERT(list);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
-	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
-	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
-	if (local_entry)
-		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
-		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
-	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
-	if (unlikely(!entry))
-		return NULL;
-	local_entry = list->cb_clone(list, entry, ctx);
-	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
-		return NULL;
-	}
-	entry->ref_cnt = 1u;
-	local_entry->ref_cnt = 1u;
-	local_entry->gentry = entry;
-	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
-	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
-							       RTE_MAX_LCORE,
-							       ctx, true);
-
-		if (unlikely(oentry)) {
-			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
-		}
-	}
-	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
-		(void *)entry, entry->ref_cnt);
-	return local_entry;
-}
-
-int
-mlx5_list_unregister(struct mlx5_list *list,
-		      struct mlx5_list_entry *entry)
-{
-	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
-
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
-	if (entry->lcore_idx == (uint32_t)lcore_idx) {
-		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
-	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
-				   __ATOMIC_RELAXED);
-	} else {
-		return 0;
-	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	rte_rwlock_write_lock(&list->lock);
-	if (likely(gentry->ref_cnt == 0)) {
-		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
-		return 0;
-	}
-	rte_rwlock_write_unlock(&list->lock);
-	return 1;
-}
-
-void
-mlx5_list_destroy(struct mlx5_list *list)
-{
-	struct mlx5_list_entry *entry;
-	int i;
-
-	MLX5_ASSERT(list);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
-			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
-					(void *)entry);
-			} else {
-				list->cb_clone_free(list, entry);
-			}
-		}
-	}
-	mlx5_free(list);
-}
-
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list)
-{
-	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
-}
-
 /********************* Indexed pool **********************/
 
 static inline void
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index ea64bb75c9..cf3db89403 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -297,203 +297,6 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ mlx5 list *****************************/
-
-/** Maximum size of string for naming. */
-#define MLX5_NAME_SIZE			32
-
-struct mlx5_list;
-
-/**
- * Structure of the entry in the mlx5 list, user should define its own struct
- * that contains this in order to store the data.
- */
-struct mlx5_list_entry {
-	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* 0 means, entry is invalid. */
-	uint32_t lcore_idx;
-	struct mlx5_list_entry *gentry;
-};
-
-struct mlx5_list_cache {
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
-	uint32_t inv_cnt; /* Invalid entries counter. */
-} __rte_cache_aligned;
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
-				   struct mlx5_list_entry *entry, void *ctx);
-
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
-
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined mlx5 list entry creation.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer of entry on success, NULL otherwise.
- */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
-
-/**
- * Linked mlx5 list structure.
- *
- * Entry in mlx5 list could be reused if entry already exists,
- * reference count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- *
- * Linked list is designed for limited number of entries,
- * read mostly, less modification.
- *
- * For huge amount of entries, please consider hash list.
- *
- */
-struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
-	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-};
-
-/**
- * Create a mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param name
- *   Name of the mlx5 list.
- * @param ctx
- *   Pointer to the list context data.
- * @param cb_create
- *   Callback function for entry create.
- * @param cb_match
- *   Callback function for entry match.
- * @param cb_remove
- *   Callback function for entry remove.
- * @return
- *   List pointer on success, otherwise NULL.
- */
-struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
-				   mlx5_list_create_cb cb_create,
-				   mlx5_list_match_cb cb_match,
-				   mlx5_list_remove_cb cb_remove,
-				   mlx5_list_clone_cb cb_clone,
-				   mlx5_list_clone_free_cb cb_clone_free);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param list
- *   Pointer to the mlx5 list.
- * @param ctx
- *   Common context parameter used by entry callback function.
- *
- * @return
- *   Pointer of the list entry if found, NULL otherwise.
- */
-struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
-					   void *ctx);
-
-/**
- * Reuse or create an entry to the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param ctx
- *   Common context parameter used by callback function.
- *
- * @return
- *   registered entry on success, NULL otherwise
- */
-struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
-					     void *ctx);
-
-/**
- * Remove an entry from the mlx5 list.
- *
- * User should guarantee the validity of the entry.
- *
- * @param list
- *   Pointer to the hast list.
- * @param entry
- *   Entry to be removed from the mlx5 list table.
- * @return
- *   0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_list_unregister(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry);
-
-/**
- * Destroy the mlx5 list.
- *
- * @param list
- *   Pointer to the mlx5 list.
- */
-void mlx5_list_destroy(struct mlx5_list *list);
-
-/**
- * Get entry number from the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list.
- * @return
- *   mlx5 list entry number.
- */
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list);
-
 /********************************* indexed pool *************************/
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v5 14/26] common/mlx5: add list lcore share

[Suanming Mou]

As some actions in SW-steering is only memory and can be allowed to
create duplicate objects, for lists which no need to check if there
are existing same objects in other sub local lists, search the object
only in local list will be more efficient.

This commit adds the lcore share mode to list optimized the list
register.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 46 +++++++++++++++++++------
 drivers/common/mlx5/mlx5_common_utils.h | 16 ++++++---
 drivers/net/mlx5/linux/mlx5_os.c        | 11 +++---
 drivers/net/mlx5/mlx5_flow_dv.c         |  2 +-
 drivers/net/mlx5/windows/mlx5_os.c      |  2 +-
 5 files changed, 55 insertions(+), 22 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 8bb8a6016d..6ac78ba97f 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -14,7 +14,7 @@
 /********************* mlx5 list ************************/
 
 struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
@@ -35,6 +35,7 @@ mlx5_list_create(const char *name, void *ctx,
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
+	list->lcores_share = lcores_share;
 	list->cb_create = cb_create;
 	list->cb_match = cb_match;
 	list->cb_remove = cb_remove;
@@ -119,7 +120,10 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
+			if (list->lcores_share)
+				list->cb_clone_free(list, entry);
+			else
+				list->cb_remove(list, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -145,25 +149,36 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
 		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
 	/* 3. Prepare new entry for global list and for cache. */
 	entry = list->cb_create(list, entry, ctx);
 	if (unlikely(!entry))
 		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+		return entry;
+	}
 	local_entry = list->cb_clone(list, entry, ctx);
 	if (unlikely(!local_entry)) {
 		list->cb_remove(list, entry);
 		return NULL;
 	}
-	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
@@ -207,13 +222,22 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
+		if (list->lcores_share)
+			list->cb_clone_free(list, entry);
+		else
+			list->cb_remove(list, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 96add6d003..000279d236 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -100,11 +100,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -112,17 +109,27 @@ struct mlx5_list {
 	mlx5_list_clone_free_cb cb_clone_free;
 	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
 };
 
 /**
  * Create a mlx5 list.
  *
+ * For actions in SW-steering is only memory and  can be allowed
+ * to create duplicate objects, the lists don't need to check if
+ * there are existing same objects in other sub local lists,
+ * search the object only in local list will be more efficient.
+ *
  * @param list
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
  * @param ctx
  *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
@@ -134,6 +141,7 @@ struct mlx5_list {
  */
 __rte_internal
 struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
 				   mlx5_list_create_cb cb_create,
 				   mlx5_list_match_cb cb_match,
 				   mlx5_list_remove_cb cb_remove,
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 2a9a6c3bf8..ce41fb34a0 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 						   flow_dv_port_id_create_cb,
 						   flow_dv_port_id_match_cb,
 						   flow_dv_port_id_remove_cb,
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 						    flow_dv_push_vlan_create_cb,
 						    flow_dv_push_vlan_match_cb,
 						    flow_dv_push_vlan_remove_cb,
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 						  flow_dv_sample_remove_cb,
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -1759,7 +1759,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
 				       mlx5_hrxq_clone_cb,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 5a536e3dff..4a45172a12 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -10054,7 +10054,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
 					      flow_dv_matcher_remove_cb,
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index e6176e70d2..a04f93e1d4 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

Re: [dpdk-dev] [PATCH v5 14/26] common/mlx5: add list lcore share

[Raslan Darawsheh]

Hi Suanming,

This patch will cause the following failure in compilation with CLANG :
[1443/3183] Compiling C object drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o
FAILED: drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o
clang -Idrivers/libtmp_rte_common_mlx5.a.p -Idrivers -I../../root/dpdk/drivers -Idrivers/common/mlx5 -I../../root/dpdk/drivers/common/mlx5 -Idrivers/common/mlx5/linux -I../../root/dpdk/drivers/common/mlx5/linux -Ilib/hash -I../../root/dpdk/lib/hash -I. -I../../root/dpdk -Iconfig -I../../root/dpdk/config -Ilib/eal/include -I../../root/dpdk/lib/eal/include -Ilib/eal/linux/include -I../../root/dpdk/lib/eal/linux/include -Ilib/eal/x86/include -I../../root/dpdk/lib/eal/x86/include -Ilib/eal/common -I../../root/dpdk/lib/eal/common -Ilib/eal -I../../root/dpdk/lib/eal -Ilib/kvargs -I../../root/dpdk/lib/kvargs -Ilib/metrics -I../../root/dpdk/lib/metrics -Ilib/telemetry -I../../root/dpdk/lib/telemetry -Ilib/net -I../../root/dpdk/lib/net -Ilib/mbuf -I../../root/dpdk/lib/mbuf -Ilib/mempool -I../../root/dpdk/lib/mempool -Ilib/ring -I../../root/dpdk/lib/ring -Ilib/rcu -I../../root/dpdk/lib/rcu -Ilib/pci -I../../root/dpdk/lib/pci -Idrivers/bus/pci -I../../root/dpdk/drivers/bus/pci -I../../root/dpdk/drivers/bus/pci/linux -I/usr/usr/include -Xclang -fcolor-diagnostics -pipe -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g -include rte_config.h -Wextra -Wcast-qual -Wdeprecated -Wformat -Wformat-nonliteral -Wformat-security -Wmissing-declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition -Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwrite-strings -Wno-address-of-packed-member -Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native -DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -std=c11 -Wno-strict-prototypes -D_BSD_SOURCE -D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -pedantic -DPEDANTIC -DRTE_LOG_DEFAULT_LOGTYPE=pmd.common.mlx5 -MD -MQ drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o -MF drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o.d -o drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o -c ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:152:6: error: variable 'entry' is used uninitialized whenever 'if' condition is false [-Werror,-Wsometimes-uninitialized]
        if (list->lcores_share) {
            ^~~~~~~~~~~~~~~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:165:32: note: uninitialized use occurs here
        entry = list->cb_create(list, entry, ctx);
                                      ^~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:152:2: note: remove the 'if' if its condition is always true
        if (list->lcores_share) {
        ^~~~~~~~~~~~~~~~~~~~~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:136:31: note: initialize the variable 'entry' to silence this warning
        struct mlx5_list_entry *entry, *local_entry;
                                     ^
                                      = NULL
1 error generated.
[1500/3183] Compiling C object drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_tx_empw.c.o
ninja: build stopped: subcommand failed.


########################
Build failed!
        CC: clang version 12.0.0 (Fedora 12.0.0-2.fc34)


Kindest regards,
Raslan Darawsheh

> -----Original Message-----
> From: Suanming Mou <suanmingm@nvidia.com>
> Sent: Monday, July 12, 2021 4:47 AM
> To: Slava Ovsiienko <viacheslavo@nvidia.com>; Matan Azrad
> <matan@nvidia.com>
> Cc: Raslan Darawsheh <rasland@nvidia.com>; Ori Kam <orika@nvidia.com>;
> dev@dpdk.org
> Subject: [PATCH v5 14/26] common/mlx5: add list lcore share
> 
> As some actions in SW-steering is only memory and can be allowed to
> create duplicate objects, for lists which no need to check if there
> are existing same objects in other sub local lists, search the object
> only in local list will be more efficient.
> 
> This commit adds the lcore share mode to list optimized the list
> register.
> 
> Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  drivers/common/mlx5/mlx5_common_utils.c | 46 +++++++++++++++++++-
> -----
>  drivers/common/mlx5/mlx5_common_utils.h | 16 ++++++---
>  drivers/net/mlx5/linux/mlx5_os.c        | 11 +++---
>  drivers/net/mlx5/mlx5_flow_dv.c         |  2 +-
>  drivers/net/mlx5/windows/mlx5_os.c      |  2 +-
>  5 files changed, 55 insertions(+), 22 deletions(-)
> 
> diff --git a/drivers/common/mlx5/mlx5_common_utils.c
> b/drivers/common/mlx5/mlx5_common_utils.c
> index 8bb8a6016d..6ac78ba97f 100644
> --- a/drivers/common/mlx5/mlx5_common_utils.c
> +++ b/drivers/common/mlx5/mlx5_common_utils.c
> @@ -14,7 +14,7 @@
>  /********************* mlx5 list ************************/
> 
>  struct mlx5_list *
> -mlx5_list_create(const char *name, void *ctx,
> +mlx5_list_create(const char *name, void *ctx, bool lcores_share,
>  		 mlx5_list_create_cb cb_create,
>  		 mlx5_list_match_cb cb_match,
>  		 mlx5_list_remove_cb cb_remove,
> @@ -35,6 +35,7 @@ mlx5_list_create(const char *name, void *ctx,
>  	if (name)
>  		snprintf(list->name, sizeof(list->name), "%s", name);
>  	list->ctx = ctx;
> +	list->lcores_share = lcores_share;
>  	list->cb_create = cb_create;
>  	list->cb_match = cb_match;
>  	list->cb_remove = cb_remove;
> @@ -119,7 +120,10 @@ __list_cache_clean(struct mlx5_list *list, int
> lcore_index)
> 
>  		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED)
> == 0) {
>  			LIST_REMOVE(entry, next);
> -			list->cb_clone_free(list, entry);
> +			if (list->lcores_share)
> +				list->cb_clone_free(list, entry);
> +			else
> +				list->cb_remove(list, entry);
>  			inv_cnt--;
>  		}
>  		entry = nentry;
> @@ -145,25 +149,36 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
>  	local_entry = __list_lookup(list, lcore_index, ctx, true);
>  	if (local_entry)
>  		return local_entry;
> -	/* 2. Lookup with read lock on global list, reuse if found. */
> -	rte_rwlock_read_lock(&list->lock);
> -	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
> -	if (likely(entry)) {
> +	if (list->lcores_share) {
> +		/* 2. Lookup with read lock on global list, reuse if found. */
> +		rte_rwlock_read_lock(&list->lock);
> +		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
> +		if (likely(entry)) {
> +			rte_rwlock_read_unlock(&list->lock);
> +			return mlx5_list_cache_insert(list, lcore_index,
> entry,
> +						      ctx);
> +		}
> +		prev_gen_cnt = list->gen_cnt;
>  		rte_rwlock_read_unlock(&list->lock);
> -		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
>  	}
> -	prev_gen_cnt = list->gen_cnt;
> -	rte_rwlock_read_unlock(&list->lock);
>  	/* 3. Prepare new entry for global list and for cache. */
>  	entry = list->cb_create(list, entry, ctx);
>  	if (unlikely(!entry))
>  		return NULL;
> +	entry->ref_cnt = 1u;
> +	if (!list->lcores_share) {
> +		entry->lcore_idx = (uint32_t)lcore_index;
> +		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry,
> next);
> +		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
> +		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
> +			list->name, lcore_index, (void *)entry, entry-
> >ref_cnt);
> +		return entry;
> +	}
>  	local_entry = list->cb_clone(list, entry, ctx);
>  	if (unlikely(!local_entry)) {
>  		list->cb_remove(list, entry);
>  		return NULL;
>  	}
> -	entry->ref_cnt = 1u;
>  	local_entry->ref_cnt = 1u;
>  	local_entry->gentry = entry;
>  	local_entry->lcore_idx = (uint32_t)lcore_index;
> @@ -207,13 +222,22 @@ mlx5_list_unregister(struct mlx5_list *list,
>  	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
>  	if (entry->lcore_idx == (uint32_t)lcore_idx) {
>  		LIST_REMOVE(entry, next);
> -		list->cb_clone_free(list, entry);
> +		if (list->lcores_share)
> +			list->cb_clone_free(list, entry);
> +		else
> +			list->cb_remove(list, entry);
>  	} else if (likely(lcore_idx != -1)) {
>  		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt,
> 1,
>  				   __ATOMIC_RELAXED);
>  	} else {
>  		return 0;
>  	}
> +	if (!list->lcores_share) {
> +		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
> +		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
> +			list->name, (void *)entry);
> +		return 0;
> +	}
>  	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED)
> != 0)
>  		return 1;
>  	rte_rwlock_write_lock(&list->lock);
> diff --git a/drivers/common/mlx5/mlx5_common_utils.h
> b/drivers/common/mlx5/mlx5_common_utils.h
> index 96add6d003..000279d236 100644
> --- a/drivers/common/mlx5/mlx5_common_utils.h
> +++ b/drivers/common/mlx5/mlx5_common_utils.h
> @@ -100,11 +100,8 @@ typedef struct mlx5_list_entry
> *(*mlx5_list_create_cb)
>   */
>  struct mlx5_list {
>  	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
> -	volatile uint32_t gen_cnt;
> -	/* List modification will update generation count. */
> -	volatile uint32_t count; /* number of entries in list. */
>  	void *ctx; /* user objects target to callback. */
> -	rte_rwlock_t lock; /* read/write lock. */
> +	bool lcores_share; /* Whether to share objects between the lcores.
> */
>  	mlx5_list_create_cb cb_create; /**< entry create callback. */
>  	mlx5_list_match_cb cb_match; /**< entry match callback. */
>  	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
> @@ -112,17 +109,27 @@ struct mlx5_list {
>  	mlx5_list_clone_free_cb cb_clone_free;
>  	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
>  	/* Lcore cache, last index is the global cache. */
> +	volatile uint32_t gen_cnt; /* List modification may update it. */
> +	volatile uint32_t count; /* number of entries in list. */
> +	rte_rwlock_t lock; /* read/write lock. */
>  };
> 
>  /**
>   * Create a mlx5 list.
>   *
> + * For actions in SW-steering is only memory and  can be allowed
> + * to create duplicate objects, the lists don't need to check if
> + * there are existing same objects in other sub local lists,
> + * search the object only in local list will be more efficient.
> + *
>   * @param list
>   *   Pointer to the hast list table.
>   * @param name
>   *   Name of the mlx5 list.
>   * @param ctx
>   *   Pointer to the list context data.
> + * @param lcores_share
> + *   Whether to share objects between the lcores.
>   * @param cb_create
>   *   Callback function for entry create.
>   * @param cb_match
> @@ -134,6 +141,7 @@ struct mlx5_list {
>   */
>  __rte_internal
>  struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
> +				   bool lcores_share,
>  				   mlx5_list_create_cb cb_create,
>  				   mlx5_list_match_cb cb_match,
>  				   mlx5_list_remove_cb cb_remove,
> diff --git a/drivers/net/mlx5/linux/mlx5_os.c
> b/drivers/net/mlx5/linux/mlx5_os.c
> index 2a9a6c3bf8..ce41fb34a0 100644
> --- a/drivers/net/mlx5/linux/mlx5_os.c
> +++ b/drivers/net/mlx5/linux/mlx5_os.c
> @@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
>  #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>  	/* Init port id action list. */
>  	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
> -	sh->port_id_action_list = mlx5_list_create(s, sh,
> +	sh->port_id_action_list = mlx5_list_create(s, sh, true,
> 
> flow_dv_port_id_create_cb,
> 
> flow_dv_port_id_match_cb,
> 
> flow_dv_port_id_remove_cb,
> @@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
>  		goto error;
>  	/* Init push vlan action list. */
>  	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
> -	sh->push_vlan_action_list = mlx5_list_create(s, sh,
> +	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
> 
> flow_dv_push_vlan_create_cb,
> 
> flow_dv_push_vlan_match_cb,
> 
> flow_dv_push_vlan_remove_cb,
> @@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
>  		goto error;
>  	/* Init sample action list. */
>  	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
> -	sh->sample_action_list = mlx5_list_create(s, sh,
> +	sh->sample_action_list = mlx5_list_create(s, sh, true,
>  						  flow_dv_sample_create_cb,
>  						  flow_dv_sample_match_cb,
> 
> flow_dv_sample_remove_cb,
> @@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
>  		goto error;
>  	/* Init dest array action list. */
>  	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
> -	sh->dest_array_list = mlx5_list_create(s, sh,
> +	sh->dest_array_list = mlx5_list_create(s, sh, true,
>  					       flow_dv_dest_array_create_cb,
>  					       flow_dv_dest_array_match_cb,
>  					       flow_dv_dest_array_remove_cb,
> @@ -1759,7 +1759,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
>  			err = ENOTSUP;
>  			goto error;
>  	}
> -	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
> mlx5_hrxq_create_cb,
> +	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
> +				       mlx5_hrxq_create_cb,
>  				       mlx5_hrxq_match_cb,
>  				       mlx5_hrxq_remove_cb,
>  				       mlx5_hrxq_clone_cb,
> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c
> b/drivers/net/mlx5/mlx5_flow_dv.c
> index 5a536e3dff..4a45172a12 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -10054,7 +10054,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list,
> uint64_t key64, void *cb_ctx)
>  	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
>  	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
>  	      key.level, key.id);
> -	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
> +	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
>  					      flow_dv_matcher_create_cb,
>  					      flow_dv_matcher_match_cb,
>  					      flow_dv_matcher_remove_cb,
> diff --git a/drivers/net/mlx5/windows/mlx5_os.c
> b/drivers/net/mlx5/windows/mlx5_os.c
> index e6176e70d2..a04f93e1d4 100644
> --- a/drivers/net/mlx5/windows/mlx5_os.c
> +++ b/drivers/net/mlx5/windows/mlx5_os.c
> @@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
>  			err = ENOTSUP;
>  			goto error;
>  	}
> -	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
> +	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
>  		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
>  		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
>  		mlx5_hrxq_clone_free_cb);
> --
> 2.25.1

Re: [dpdk-dev] [PATCH v5 14/26] common/mlx5: add list lcore share

[Suanming Mou]

Hi Raslan,

> -----Original Message-----
> From: Raslan Darawsheh <rasland@nvidia.com>
> Sent: Monday, July 12, 2021 10:59 PM
> To: Suanming Mou <suanmingm@nvidia.com>; Slava Ovsiienko
> <viacheslavo@nvidia.com>; Matan Azrad <matan@nvidia.com>
> Cc: Ori Kam <orika@nvidia.com>; dev@dpdk.org
> Subject: RE: [PATCH v5 14/26] common/mlx5: add list lcore share
> 
> Hi Suanming,
> 
> This patch will cause the following failure in compilation with CLANG :
> [1443/3183] Compiling C object
> drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o
> FAILED:
> drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o
> clang -Idrivers/libtmp_rte_common_mlx5.a.p -Idrivers -I../../root/dpdk/drivers -
> Idrivers/common/mlx5 -I../../root/dpdk/drivers/common/mlx5 -
> Idrivers/common/mlx5/linux -I../../root/dpdk/drivers/common/mlx5/linux -
> Ilib/hash -I../../root/dpdk/lib/hash -I. -I../../root/dpdk -Iconfig -
> I../../root/dpdk/config -Ilib/eal/include -I../../root/dpdk/lib/eal/include -
> Ilib/eal/linux/include -I../../root/dpdk/lib/eal/linux/include -Ilib/eal/x86/include
> -I../../root/dpdk/lib/eal/x86/include -Ilib/eal/common -
> I../../root/dpdk/lib/eal/common -Ilib/eal -I../../root/dpdk/lib/eal -Ilib/kvargs -
> I../../root/dpdk/lib/kvargs -Ilib/metrics -I../../root/dpdk/lib/metrics -
> Ilib/telemetry -I../../root/dpdk/lib/telemetry -Ilib/net -I../../root/dpdk/lib/net -
> Ilib/mbuf -I../../root/dpdk/lib/mbuf -Ilib/mempool -
> I../../root/dpdk/lib/mempool -Ilib/ring -I../../root/dpdk/lib/ring -Ilib/rcu -
> I../../root/dpdk/lib/rcu -Ilib/pci -I../../root/dpdk/lib/pci -Idrivers/bus/pci -
> I../../root/dpdk/drivers/bus/pci -I../../root/dpdk/drivers/bus/pci/linux -
> I/usr/usr/include -Xclang -fcolor-diagnostics -pipe -D_FILE_OFFSET_BITS=64 -
> Wall -Winvalid-pch -Werror -O2 -g -include rte_config.h -Wextra -Wcast-qual -
> Wdeprecated -Wformat -Wformat-nonliteral -Wformat-security -Wmissing-
> declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition -
> Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwrite-strings -
> Wno-address-of-packed-member -Wno-missing-field-initializers -
> D_GNU_SOURCE -fPIC -march=native -DALLOW_EXPERIMENTAL_API -
> DALLOW_INTERNAL_API -std=c11 -Wno-strict-prototypes -D_BSD_SOURCE -
> D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -pedantic -DPEDANTIC -
> DRTE_LOG_DEFAULT_LOGTYPE=pmd.common.mlx5 -MD -MQ
> drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o
> -MF
> drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o.
> d -o
> drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o
> -c ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c
> ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:152:6: error:
> variable 'entry' is used uninitialized whenever 'if' condition is false [-Werror,-
> Wsometimes-uninitialized]
>         if (list->lcores_share) {
>             ^~~~~~~~~~~~~~~~~~
> ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:165:32: note:
> uninitialized use occurs here
>         entry = list->cb_create(list, entry, ctx);
>                                       ^~~~~
> ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:152:2: note:
> remove the 'if' if its condition is always true
>         if (list->lcores_share) {
>         ^~~~~~~~~~~~~~~~~~~~~~~~
> ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:136:31: note:
> initialize the variable 'entry' to silence this warning
>         struct mlx5_list_entry *entry, *local_entry;
>                                      ^
>                                       = NULL
> 1 error generated.
> [1500/3183] Compiling C object
> drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_tx_empw.c.o
> ninja: build stopped: subcommand failed.
> 
> 
> ########################
> Build failed!
>         CC: clang version 12.0.0 (Fedora 12.0.0-2.fc34)

OK, I will check it.

> 
> 
> Kindest regards,
> Raslan Darawsheh
> 

[dpdk-dev] [PATCH v5 15/26] common/mlx5: call list callbacks with context

[Suanming Mou]

This commit optimizes to call the list callback functions with global
context directly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c |  24 ++---
 drivers/common/mlx5/mlx5_common_utils.h |  36 ++++----
 drivers/net/mlx5/mlx5_flow.h            |  94 +++++++++----------
 drivers/net/mlx5/mlx5_flow_dv.c         | 115 +++++++++++-------------
 drivers/net/mlx5/mlx5_rx.h              |  13 ++-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +++--------
 6 files changed, 132 insertions(+), 203 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 6ac78ba97f..50c98143f7 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -55,7 +55,7 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
+		if (list->cb_match(list->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
@@ -96,7 +96,7 @@ static struct mlx5_list_entry *
 mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
@@ -121,9 +121,9 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
 			if (list->lcores_share)
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			else
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -162,7 +162,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_rwlock_read_unlock(&list->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
+	entry = list->cb_create(list->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
@@ -174,9 +174,9 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list, entry, ctx);
+	local_entry = list->cb_clone(list->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
+		list->cb_remove(list->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
@@ -192,8 +192,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
+			list->cb_remove(list->ctx, entry);
+			list->cb_clone_free(list->ctx, local_entry);
 			return mlx5_list_cache_insert(list, lcore_index, oentry,
 						      ctx);
 		}
@@ -223,9 +223,9 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (list->lcores_share)
-			list->cb_clone_free(list, entry);
+			list->cb_clone_free(list->ctx, entry);
 		else
-			list->cb_remove(list, entry);
+			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
@@ -244,7 +244,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
+		list->cb_remove(list->ctx, gentry);
 		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 000279d236..a691ff8f0d 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -33,19 +33,19 @@ struct mlx5_list_cache {
 /**
  * Type of callback function for entry removal.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance user context.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_remove_cb)(void *tool_ctx,
+				    struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance context.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -54,34 +54,28 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+typedef int (*mlx5_list_match_cb)(void *tool_ctx,
 				   struct mlx5_list_entry *entry, void *ctx);
 
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)(void *tool_ctx,
+				      struct mlx5_list_entry *entry, void *ctx);
 
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
+					struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
+ * @param tool_ctx
+ *   The mlx5 tool instance context.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
+						       void *ctx);
 
 /**
  * Linked mlx5 list structure.
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 59a6f31d2e..70b3119d09 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1631,64 +1631,50 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
 
-int flow_dv_matcher_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *ctx);
-struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
-
-int flow_dv_port_id_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused,
-				 void *cb_ctx);
-void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused);
-int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
-				(struct mlx5_list *list,
+int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_matcher_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(void *tool_ctx,
+				struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry);
+
+int flow_dv_push_vlan_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			       void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(void *tool_ctx,
+						    void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb(void *tool_ctx,
 				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
-						 struct mlx5_list_entry *entry,
-						 void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_sample_clone_cb
-				(struct mlx5_list *list,
+void flow_dv_push_vlan_clone_free_cb(void *tool_ctx,
+				     struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_sample_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb(void *tool_ctx,
 				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+void flow_dv_sample_clone_free_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_dest_array_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+
+int flow_dv_dest_array_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+				void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(void *tool_ctx,
+						     void *cb_ctx);
+void flow_dv_dest_array_remove_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(void *tool_ctx,
+				      struct mlx5_list_entry *entry);
+
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 4a45172a12..795de65089 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3780,23 +3780,21 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->port_id != res->port_id;
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
@@ -3827,11 +3825,11 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_clone_cb(void *tool_ctx,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
@@ -3849,12 +3847,11 @@ flow_dv_port_id_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
@@ -3899,23 +3896,21 @@ flow_dv_port_id_action_resource_register
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
+			   struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
@@ -3952,11 +3947,11 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_clone_cb(void *tool_ctx,
+			   struct mlx5_list_entry *entry __rte_unused,
+			   void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx;
@@ -3974,12 +3969,11 @@ flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
@@ -9959,7 +9953,7 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 }
 
 static struct mlx5_list_entry *
-flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -9982,7 +9976,7 @@ flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 static void
-flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
@@ -10225,7 +10219,7 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10240,11 +10234,9 @@ flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_matcher_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
 	struct mlx5_flow_dv_matcher *resource;
@@ -10740,7 +10732,7 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
 			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10769,9 +10761,7 @@ flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -10858,7 +10848,7 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
@@ -10884,16 +10874,15 @@ flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry)
+flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			resource->idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
 }
 
 /**
@@ -10936,14 +10925,14 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
 			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_dv_dest_array_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
 	if (ctx_resource->num_of_dest == resource->num_of_dest &&
@@ -10965,9 +10954,7 @@ flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11072,9 +11059,9 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
+			    struct mlx5_list_entry *entry __rte_unused,
+			    void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11100,8 +11087,8 @@ flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
+				 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
 			container_of(entry, typeof(*resource), entry);
@@ -13588,7 +13575,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
 			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_matcher *resource = container_of(entry,
@@ -13730,10 +13717,9 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
 				  container_of(entry, typeof(*resource), entry);
 
@@ -13787,10 +13773,9 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
 			container_of(entry, typeof(*resource), entry);
 
@@ -13859,7 +13844,7 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
@@ -13907,7 +13892,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 5450ddd388..3f2b99fb65 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index aa9e973d10..7893b3edd4 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,25 +2093,10 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Match an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   0 if match, none zero if not match.
- */
 int
-mlx5_hrxq_match_cb(struct mlx5_list *list,
-		   struct mlx5_list_entry *entry,
-		   void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2251,10 +2236,9 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
 	__mlx5_hrxq_remove(dev, hrxq);
@@ -2305,25 +2289,10 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
 	return NULL;
 }
 
-/**
- * Create an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   queue entry on success, NULL otherwise.
- */
 struct mlx5_list_entry *
-mlx5_hrxq_create_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry __rte_unused,
-		    void *cb_ctx)
+mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq;
@@ -2333,11 +2302,10 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-mlx5_hrxq_clone_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry,
+mlx5_hrxq_clone_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		    void *cb_ctx __rte_unused)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	uint32_t hrxq_idx = 0;
@@ -2351,10 +2319,9 @@ mlx5_hrxq_clone_cb(struct mlx5_list *list,
 }
 
 void
-mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
-- 
2.25.1

[dpdk-dev] [PATCH v5 16/26] common/mlx5: add per-lcore cache to hash list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates
all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and
hlist.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 238 +++++++------------
 drivers/common/mlx5/mlx5_common_utils.h | 141 +++---------
 drivers/net/mlx5/linux/mlx5_os.c        |  35 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++++++----
 drivers/net/mlx5/mlx5_flow.h            |  92 ++++----
 drivers/net/mlx5/mlx5_flow_dv.c         | 292 +++++++++++++++---------
 8 files changed, 482 insertions(+), 482 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 50c98143f7..4e385c616a 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -13,25 +13,21 @@
 
 /********************* mlx5 list ************************/
 
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx, bool lcores_share,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
+static int
+mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
+	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       mlx5_list_match_cb cb_match,
+	       mlx5_list_remove_cb cb_remove,
+	       mlx5_list_clone_cb cb_clone,
+	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	struct mlx5_list *list;
 	int i;
 
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
-		return NULL;
+		return -EINVAL;
 	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -45,6 +41,28 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
+	return 0;
+}
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (mlx5_list_init(list, name, ctx, lcores_share,
+			   cb_create, cb_match, cb_remove, cb_clone,
+			   cb_clone_free) != 0) {
+		mlx5_free(list);
+		return NULL;
+	}
 	return list;
 }
 
@@ -254,8 +272,8 @@ mlx5_list_unregister(struct mlx5_list *list,
 	return 1;
 }
 
-void
-mlx5_list_destroy(struct mlx5_list *list)
+static void
+mlx5_list_uninit(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
 	int i;
@@ -266,15 +284,21 @@ mlx5_list_destroy(struct mlx5_list *list)
 			entry = LIST_FIRST(&list->cache[i].h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", list->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			}
 		}
 	}
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	mlx5_list_uninit(list);
 	mlx5_free(list);
 }
 
@@ -287,37 +311,24 @@ mlx5_list_get_entry_num(struct mlx5_list *list)
 
 /********************* Hash List **********************/
 
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
 struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
+		  bool lcores_share, void *ctx, mlx5_list_create_cb cb_create,
+		  mlx5_list_match_cb cb_match,
+		  mlx5_list_remove_cb cb_remove,
+		  mlx5_list_clone_cb cb_clone,
+		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+		DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
+			"be aligned to 0x%" PRIX32 ".", size, act_size);
 	} else {
 		act_size = size;
 	}
@@ -331,61 +342,24 @@ mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 			name ? name : "None");
 		return NULL;
 	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
 	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
+	h->lcores_share = lcores_share;
+	h->direct_key = direct_key;
+	for (i = 0; i < act_size; i++) {
+		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   cb_create, cb_match, cb_remove, cb_clone,
+				   cb_clone_free) != 0) {
+			mlx5_free(h);
+			return NULL;
 		}
 	}
-	return node;
+	DRV_LOG(DEBUG, "Hash list %s with size 0x%" PRIX32 " was created.",
+		name, act_size);
+	return h;
 }
 
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
 
-struct mlx5_hlist_entry *
+struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
@@ -394,102 +368,44 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
+	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
 }
 
-struct mlx5_hlist_entry*
+struct mlx5_list_entry*
 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry;
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
+	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	if (likely(entry)) {
+		if (h->lcores_share)
+			entry->gentry->bucket_idx = idx;
+		else
+			entry->bucket_idx = idx;
 	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
 	return entry;
 }
 
 int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
+	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+							      entry->bucket_idx;
+
+	return mlx5_list_unregister(&h->buckets[idx].l, entry);
 }
 
 void
 mlx5_hlist_destroy(struct mlx5_hlist *h)
 {
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
+	uint32_t i;
 
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
+	for (i = 0; i <= h->mask; i++)
+		mlx5_list_uninit(&h->buckets[i].l);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index a691ff8f0d..4bb974fa3e 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -20,10 +20,13 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t ref_cnt __rte_aligned(8); /* 0 means, entry is invalid. */
 	uint32_t lcore_idx;
-	struct mlx5_list_entry *gentry;
-};
+	union {
+		struct mlx5_list_entry *gentry;
+		uint32_t bucket_idx;
+	};
+} __rte_packed;
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
@@ -212,108 +215,24 @@ __rte_internal
 uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list);
 
-/************************ Hash list *****************************/
-
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
-
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE			32
-
-struct mlx5_hlist;
-
-/**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
- */
-struct mlx5_hlist_entry {
-	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint32_t idx; /* Bucket index the entry belongs to. */
-	uint32_t ref_cnt; /* Reference count. */
-};
-
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
-				     struct mlx5_hlist_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- * @param key
- *   The new entry key.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry,
-				   uint64_t key, void *ctx);
-
-/**
- * Type of function for user defined hash list entry creation.
- *
- * @param list
- *   The hash list.
- * @param key
- *   The key of the new entry.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer to allocated entry on success, NULL otherwise.
- */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
-				  (struct mlx5_hlist *list,
-				   uint64_t key, void *ctx);
+/********************* Hash List **********************/
 
-/* Hash list bucket head. */
+/* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_hlist_head head; /* List head. */
-	rte_rwlock_t lock; /* Bucket lock. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
+	struct mlx5_list l;
 } __rte_cache_aligned;
 
 /**
  * Hash list table structure
  *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
+ * The hash list bucket using the mlx5_list object for managing.
  */
 struct mlx5_hlist {
-	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
-	/**< number of heads, need to be power of 2. */
-	uint32_t table_sz;
-	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
-	/**< mask to get the index of the list heads. */
-	uint32_t mask;
-	bool direct_key; /* Use the new entry key directly as hash index. */
-	bool write_most; /* List mostly used for append new or destroy. */
-	void *ctx;
-	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
-	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
-	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+	uint32_t mask; /* A mask for the bucket index range. */
+	uint8_t flags;
+	bool direct_key; /* Whether to use the key directly as hash index. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
-	/**< list bucket arrays. */
 };
 
 /**
@@ -330,23 +249,33 @@ struct mlx5_hlist {
  *   Heads array size of the hash list.
  * @param entry_size
  *   Entry size to allocate if cb_create not specified.
- * @param flags
- *   The hash list attribute flags.
+ * @param direct key
+ *   Whether to use the key directly as hash index.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
+ * @param ctx
+ *   The hlist instance context.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
  *   Callback function for entry match.
- * @param cb_destroy
- *   Callback function for entry destroy.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @param cb_clone
+ *   Callback function for entry clone.
+ * @param cb_clone_free
+ *   Callback function for entry clone free.
  * @return
  *   Pointer of the hash list table created, NULL on failure.
  */
 __rte_internal
 struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
-				     uint32_t entry_size, uint32_t flags,
-				     mlx5_hlist_create_cb cb_create,
-				     mlx5_hlist_match_cb cb_match,
-				     mlx5_hlist_remove_cb cb_destroy);
+				     bool direct_key, bool lcores_share,
+				     void *ctx, mlx5_list_create_cb cb_create,
+				     mlx5_list_match_cb cb_match,
+				     mlx5_list_remove_cb cb_remove,
+				     mlx5_list_clone_cb cb_clone,
+				     mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
@@ -365,7 +294,7 @@ struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
  *   Pointer of the hlist entry if found, NULL otherwise.
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
 					   void *ctx);
 
 /**
@@ -384,7 +313,7 @@ struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
  *   registered entry on success, NULL otherwise
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
 					     void *ctx);
 
 /**
@@ -399,7 +328,7 @@ struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
  *   0 on entry removed, 1 on entry still referenced.
  */
 __rte_internal
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry);
 
 /**
  * Destroy the hash list table, all the entries already inserted into the lists
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index ce41fb34a0..b04e5f8ba6 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -261,7 +261,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+	char s[MLX5_NAME_SIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -314,44 +314,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb,
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
+					  false, sh, flow_dv_tag_create_cb,
 					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb);
+					  flow_dv_tag_remove_cb,
+					  flow_dv_tag_clone_cb,
+					  flow_dv_tag_clone_free_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, MLX5_HLIST_WRITE_MOST |
-					    MLX5_HLIST_DIRECT_KEY,
+					    true, false, sh,
 					    flow_dv_modify_create_cb,
 					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb);
+					    flow_dv_modify_remove_cb,
+					    flow_dv_modify_clone_cb,
+					    flow_dv_modify_clone_free_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, MLX5_HLIST_DIRECT_KEY |
-					      MLX5_HLIST_WRITE_MOST,
+					      true, true, sh,
 					      flow_dv_encap_decap_create_cb,
 					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb);
+					      flow_dv_encap_decap_remove_cb,
+					      flow_dv_encap_decap_clone_cb,
+					     flow_dv_encap_decap_clone_free_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -1792,15 +1792,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
-						      0, 0,
+						      false, true, eth_dev,
 						      flow_dv_mreg_create_cb,
 						      flow_dv_mreg_match_cb,
-						      flow_dv_mreg_remove_cb);
+						      flow_dv_mreg_remove_cb,
+						      flow_dv_mreg_clone_cb,
+						    flow_dv_mreg_clone_free_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
-		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index d04b5d616f..fd675c9262 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1359,20 +1359,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 88b84d749a..e9b08094a6 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index d62361362e..45e90bb245 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -4040,28 +4040,27 @@ flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
-flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
-		      struct mlx5_hlist_entry *entry,
-		      uint64_t key, void *cb_ctx __rte_unused)
+flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
+		      struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
 
-	return mcp_res->mark_id != key;
+	return mcp_res->mark_id != *(uint32_t *)(ctx->data);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
-		       void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
 	struct rte_flow_error *error = ctx->error;
 	uint32_t idx = 0;
 	int ret;
-	uint32_t mark_id = key;
+	uint32_t mark_id = *(uint32_t *)(ctx->data);
 	struct rte_flow_attr attr = {
 		.group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
 		.ingress = 1,
@@ -4167,6 +4166,36 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return &mcp_res->hlist_ent;
 }
 
+struct mlx5_list_entry *
+flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	uint32_t idx = 0;
+
+	mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
+	if (!mcp_res) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	memcpy(mcp_res, oentry, sizeof(*mcp_res));
+	mcp_res->idx = idx;
+	return &mcp_res->hlist_ent;
+}
+
+void
+flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_mreg_copy_resource *mcp_res =
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
+}
+
 /**
  * Add a flow of copying flow metadata registers in RX_CP_TBL.
  *
@@ -4197,10 +4226,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 			  struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
+		.data = &mark_id,
 	};
 
 	/* Check if already registered. */
@@ -4213,11 +4243,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 }
 
 void
-flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
+flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
-	struct rte_eth_dev *dev = list->ctx;
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
@@ -4263,14 +4293,17 @@ flow_mreg_del_copy_action(struct rte_eth_dev *dev,
 static void
 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
 {
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check if default flow is registered. */
 	if (!priv->mreg_cp_tbl)
 		return;
-	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
-				  MLX5_DEFAULT_COPY_ID, NULL);
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
 	if (!entry)
 		return;
 	mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
@@ -4296,6 +4329,8 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check whether extensive metadata feature is engaged. */
 	if (!priv->config.dv_flow_en ||
@@ -4307,9 +4342,11 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 	 * Add default mreg copy flow may be called multiple time, but
 	 * only be called once in stop. Avoid register it twice.
 	 */
-	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
 		return 0;
-	mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
+	mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
 	if (!mcp_res)
 		return -rte_errno;
 	return 0;
@@ -8606,7 +8643,7 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	union tunnel_offload_mark mbits = { .val = mark };
 	union mlx5_flow_tbl_key table_key = {
 		{
@@ -8618,16 +8655,20 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 			.is_egress = 0,
 		}
 	};
-	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &table_key.v64,
+	};
+
+	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
 	return he ?
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
 
 static void
-mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry)
+mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
@@ -8636,26 +8677,26 @@ mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
 }
 
 static int
-mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-				  struct mlx5_hlist_entry *entry,
-				  uint64_t key, void *cb_ctx __rte_unused)
+mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
 }
 
-static struct mlx5_hlist_entry *
-mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
-				   void *ctx __rte_unused)
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 
 	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
@@ -8684,13 +8725,36 @@ mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return NULL;
 }
 
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *oentry,
+				  void *cb_ctx __rte_unused)
+{
+	struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
+						   0, SOCKET_ID_ANY);
+
+	if (!tte)
+		return NULL;
+	memcpy(tte, oentry, sizeof(*tte));
+	return &tte->hash;
+}
+
+static void
+mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
+				       struct mlx5_list_entry *entry)
+{
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	mlx5_free(tte);
+}
+
 static uint32_t
 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 				const struct mlx5_flow_tunnel *tunnel,
 				uint32_t group, uint32_t *table,
 				struct rte_flow_error *error)
 {
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key key = {
 		.tunnel_id = tunnel ? tunnel->tunnel_id : 0,
@@ -8698,9 +8762,12 @@ tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 	};
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_hlist *group_hash;
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &key.val,
+	};
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_register(group_hash, key.val, NULL);
+	he = mlx5_hlist_register(group_hash, key.val, &ctx);
 	if (!he)
 		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
@@ -8814,15 +8881,17 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
-					   mlx5_flow_tunnel_grp2tbl_remove_cb);
+					   mlx5_flow_tunnel_grp2tbl_remove_cb,
+					   mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(ipool, id);
 		return NULL;
 	}
-	tunnel->groups->ctx = priv->sh;
 	/* initiate new PMD tunnel */
 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
 	tunnel->tunnel_id = id;
@@ -8922,15 +8991,17 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES), 0,
-					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
+					 rte_align32pow2(MLX5_MAX_TABLES),
+					 false, true, sh,
+					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-					 mlx5_flow_tunnel_grp2tbl_remove_cb);
+					 mlx5_flow_tunnel_grp2tbl_remove_cb,
+					 mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
 	}
-	thub->groups->ctx = sh;
 	sh->tunnel_hub = thub;
 
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 70b3119d09..e3a29297ba 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -481,7 +481,7 @@ struct mlx5_flow_dv_matcher {
 
 /* Encap/decap resource structure. */
 struct mlx5_flow_dv_encap_decap_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/* Pointer to next element. */
 	uint32_t refcnt; /**< Reference counter. */
 	void *action;
@@ -496,7 +496,7 @@ struct mlx5_flow_dv_encap_decap_resource {
 
 /* Tag resource structure. */
 struct mlx5_flow_dv_tag_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry for tag resource, tag value as the key. */
 	void *action;
 	/**< Tag action object. */
@@ -520,7 +520,7 @@ struct mlx5_flow_dv_tag_resource {
 
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
@@ -570,7 +570,7 @@ struct mlx5_flow_mreg_copy_resource {
 	 *  - Key is 32/64-bit MARK action ID.
 	 *  - MUST be the first entry.
 	 */
-	struct mlx5_hlist_entry hlist_ent;
+	struct mlx5_list_entry hlist_ent;
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
 	uint32_t idx;
@@ -587,7 +587,7 @@ struct mlx5_flow_tbl_tunnel_prm {
 
 /* Table data structure of the hash organization. */
 struct mlx5_flow_tbl_data_entry {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
@@ -927,7 +927,7 @@ struct mlx5_flow_tunnel_hub {
 
 /* convert jump group to flow table ID in tunnel rules */
 struct tunnel_tbl_entry {
-	struct mlx5_hlist_entry hash;
+	struct mlx5_list_entry hash;
 	uint32_t flow_table;
 	uint32_t tunnel_id;
 	uint32_t group;
@@ -1585,51 +1585,59 @@ int mlx5_action_handle_flush(struct rte_eth_dev *dev);
 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id);
 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh);
 
-/* Hash list callbacks for flow tables: */
-struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *entry_ctx);
-int flow_dv_tbl_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tbl_create_cb(void *tool_ctx, void *entry_ctx);
+int flow_dv_tbl_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
+void flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tbl_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *entry_ctx);
+void flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
 struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 		uint32_t table_level, uint8_t egress, uint8_t transfer,
 		bool external, const struct mlx5_flow_tunnel *tunnel,
 		uint32_t group_id, uint8_t dummy,
 		uint32_t table_id, struct rte_flow_error *error);
 
-struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *cb_ctx);
-int flow_dv_tag_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx);
+int flow_dv_tag_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
-
-int flow_dv_modify_match_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry,
-			    uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
-						  uint64_t key, void *ctx);
-void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry);
-
-struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
-						uint64_t key, void *ctx);
-int flow_dv_mreg_match_cb(struct mlx5_hlist *list,
-			  struct mlx5_hlist_entry *entry, uint64_t key,
+void flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tag_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *cb_ctx);
+void flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_modify_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_modify_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_modify_clone_cb(void *tool_ctx,
+						struct mlx5_list_entry *oentry,
+						void *ctx);
+void flow_dv_modify_clone_free_cb(void *tool_ctx,
+				  struct mlx5_list_entry *entry);
+
+struct mlx5_list_entry *flow_dv_mreg_create_cb(void *tool_ctx, void *ctx);
+int flow_dv_mreg_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			  void *cb_ctx);
-void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry);
-
-int flow_dv_encap_decap_match_cb(struct mlx5_hlist *list,
-				 struct mlx5_hlist_entry *entry,
-				 uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-				uint64_t key, void *cb_ctx);
-void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry);
+void flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_mreg_clone_cb(void *tool_ctx,
+					      struct mlx5_list_entry *entry,
+					      void *ctx);
+void flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_encap_decap_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+				 void *cb_ctx);
+struct mlx5_list_entry *flow_dv_encap_decap_create_cb(void *tool_ctx,
+						      void *cb_ctx);
+void flow_dv_encap_decap_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_encap_decap_clone_cb(void *tool_ctx,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_encap_decap_clone_free_cb(void *tool_ctx,
+				       struct mlx5_list_entry *entry);
 
 int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			     void *ctx);
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 795de65089..4b9fd22824 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3586,25 +3586,9 @@ flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
 	return 0;
 }
 
-/**
- * Match encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
 int
-flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
-			     struct mlx5_hlist_entry *entry,
-			     uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3623,25 +3607,10 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 	return -1;
 }
 
-/**
- * Allocate encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
-struct mlx5_hlist_entry *
-flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-			      uint64_t key __rte_unused,
-			      void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3679,6 +3648,38 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			     void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	uint32_t idx;
+
+	cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+					   &idx);
+	if (!cache_resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(cache_resource, oentry, sizeof(*cache_resource));
+	cache_resource->idx = idx;
+	return &cache_resource->entry;
+}
+
+void
+flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *res =
+				       container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
+}
+
 /**
  * Find existing encap/decap resource or create and register a new one.
  *
@@ -3703,7 +3704,7 @@ flow_dv_encap_decap_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	union {
 		struct {
 			uint32_t ft_type:8;
@@ -5328,30 +5329,14 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
 	return ret;
 }
 
-/**
- * Match modify-header resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx
- *   Pointer to new modify-header resource.
- *
- * @return
- *   0 on matching, non-zero otherwise.
- */
 int
-flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
-			struct mlx5_hlist_entry *entry,
-			uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	struct mlx5_flow_dv_modify_hdr_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
 
 	key_len += ref->actions_num * sizeof(ref->actions[0]);
@@ -5359,11 +5344,10 @@ flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
-			 void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
@@ -5402,6 +5386,33 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
 	return &entry->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *oentry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *entry;
+	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+
+	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	return &entry->entry;
+}
+
+void
+flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 /**
  * Validate the sample action.
  *
@@ -5673,7 +5684,7 @@ flow_dv_modify_hdr_resource_register
 	uint32_t key_len = sizeof(*resource) -
 			   offsetof(typeof(*resource), ft_type) +
 			   resource->actions_num * sizeof(resource->actions[0]);
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
 		.data = resource,
@@ -9982,16 +9993,16 @@ flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
 	mlx5_free(entry);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
-	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data;
+	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
 	struct rte_flow_error *error = ctx->error;
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
 	struct mlx5_flow_tbl_resource *tbl;
 	void *domain;
 	uint32_t idx = 0;
@@ -10068,13 +10079,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 }
 
 int
-flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key64,
-		     void *cb_ctx __rte_unused)
+flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
 		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
 
 	return tbl_data->level != key.level ||
 	       tbl_data->id != key.id ||
@@ -10083,6 +10094,39 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
 	       tbl_data->is_egress != !!key.is_egress;
 }
 
+struct mlx5_list_entry *
+flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data;
+	struct rte_flow_error *error = ctx->error;
+	uint32_t idx = 0;
+
+	tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
+	if (!tbl_data) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate flow table data entry");
+		return NULL;
+	}
+	memcpy(tbl_data, oentry, sizeof(*tbl_data));
+	tbl_data->idx = idx;
+	return &tbl_data->entry;
+}
+
+void
+flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data =
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
+}
+
 /**
  * Get a flow table.
  *
@@ -10133,9 +10177,10 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = &tt_prm,
+		.data = &table_key.v64,
+		.data2 = &tt_prm,
 	};
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 
 	entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
@@ -10154,12 +10199,11 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
-		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
 
 	MLX5_ASSERT(entry && sh);
 	if (tbl_data->jump.action)
@@ -10167,7 +10211,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 	if (tbl_data->tbl.obj)
 		mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
 	if (tbl_data->tunnel_offload && tbl_data->external) {
-		struct mlx5_hlist_entry *he;
+		struct mlx5_list_entry *he;
 		struct mlx5_hlist *tunnel_grp_hash;
 		struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
 		union tunnel_tbl_key tunnel_key = {
@@ -10176,11 +10220,14 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			.group = tbl_data->group_id
 		};
 		uint32_t table_level = tbl_data->level;
+		struct mlx5_flow_cb_ctx ctx = {
+			.data = (void *)&tunnel_key.val,
+		};
 
 		tunnel_grp_hash = tbl_data->tunnel ?
 					tbl_data->tunnel->groups :
 					thub->groups;
-		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
+		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
 		if (he)
 			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
@@ -10333,29 +10380,29 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 	return 0;
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
+struct mlx5_list_entry *
+flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct rte_flow_error *error = ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_tag_resource *entry;
 	uint32_t idx = 0;
 	int ret;
 
 	entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
 	if (!entry) {
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
 	entry->idx = idx;
-	entry->tag_id = key;
-	ret = mlx5_flow_os_create_flow_action_tag(key,
+	entry->tag_id = *(uint32_t *)(ctx->data);
+	ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
 						  &entry->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create action");
 		return NULL;
@@ -10364,14 +10411,45 @@ flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
 }
 
 int
-flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key,
-		     void *cb_ctx __rte_unused)
+flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
-	return key != tag->tag_id;
+	return *(uint32_t *)(ctx->data) != tag->tag_id;
+}
+
+struct mlx5_list_entry *
+flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		     void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_tag_resource *entry;
+	uint32_t idx = 0;
+
+	entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate tag resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry));
+	entry->idx = idx;
+	return &entry->entry;
+}
+
+void
+flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_tag_resource *tag =
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
 }
 
 /**
@@ -10398,9 +10476,13 @@ flow_dv_tag_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_dv_tag_resource *resource;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_cb_ctx ctx = {
+					.error = error,
+					.data = &tag_be24,
+					};
 
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
+	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
@@ -10412,12 +10494,11 @@ flow_dv_tag_resource_register
 }
 
 void
-flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
 	MLX5_ASSERT(tag && sh && tag->action);
 	claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
@@ -13612,19 +13693,10 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Release encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- */
 void
-flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry)
+flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
 				       container_of(entry, typeof(*res), entry);
 
@@ -13684,8 +13756,8 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused,
-			 struct mlx5_hlist_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
-- 
2.25.1

[dpdk-dev] [PATCH v5 17/26] common/mlx5: allocate cache list memory individually

[Suanming Mou]

Currently, the list's local cache instance memory is allocated with
the list. As the local cache instance array size is RTE_MAX_LCORE,
most of the cases the system will only have very limited cores.
allocate the instance memory individually per core will be more
economic to the memory.

This commit changes the instance array to pointer array, allocate
the local cache memory only when the core is to be used.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 4e385c616a..f75b1cb0da 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 4bb974fa3e..d49fb64457 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-- 
2.25.1

[dpdk-dev] [PATCH v5 18/26] common/mlx5: optimize cache list object memory

[Suanming Mou]

Currently, hash list uses the cache list as bucket list. The list
in the buckets have the same name, ctx and callbacks. This wastes
the memory.

This commit abstracts all the name, ctx and callback members in the
list to a constant struct and others to the inconstant struct, uses
the wrapper functions to satisfy both hash list and cache list can
set the list constant and inconstant struct individually.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index f75b1cb0da..858c8d8164 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index d49fb64457..5718a21be0 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -80,6 +80,32 @@ typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
 typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
 						       void *ctx);
 
+/**
+ * Linked mlx5 list constant object.
+ */
+struct mlx5_list_const {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	void *ctx; /* user objects target to callback. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	/**< entry clone free callback. */
+};
+
+/**
+ * Linked mlx5 list inconstant data.
+ */
+struct mlx5_list_inconst {
+	rte_rwlock_t lock; /* read/write lock. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+};
+
 /**
  * Linked mlx5 list structure.
  *
@@ -96,19 +122,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
  *
  */
 struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	void *ctx; /* user objects target to callback. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-	volatile uint32_t gen_cnt; /* List modification may update it. */
-	volatile uint32_t count; /* number of entries in list. */
-	rte_rwlock_t lock; /* read/write lock. */
+	struct mlx5_list_const l_const;
+	struct mlx5_list_inconst l_inconst;
 };
 
 /**
@@ -219,7 +234,7 @@ mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_list l;
+	struct mlx5_list_inconst l;
 } __rte_cache_aligned;
 
 /**
@@ -231,7 +246,7 @@ struct mlx5_hlist {
 	uint32_t mask; /* A mask for the bucket index range. */
 	uint8_t flags;
 	bool direct_key; /* Whether to use the key directly as hash index. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
+	struct mlx5_list_const l_const; /* List constant data. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v5 19/26] common/mlx5: support list non-lcore operations

[Suanming Mou]

This commit supports the list non-lcore operations with
an extra sub-list and lock.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 858c8d8164..c8b48f3afc 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.lcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.lcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
+	return ret;
 }
 
 void
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 5718a21be0..613d29de0c 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t lcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v5 20/26] net/mlx5: move modify header allocator to ipool

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Modify header actions are allocated by mlx5_malloc which has a big
overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has
its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in
the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has
better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header
handler.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index fd675c9262..640d36c6be 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -802,6 +802,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -815,6 +816,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e9b08094a6..e7e4749824 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1146,6 +1159,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index e3a29297ba..7027012220 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -505,23 +505,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 4b9fd22824..e39fe43854 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5344,6 +5344,45 @@ flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
+static struct mlx5_indexed_pool *
+flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
+{
+	struct mlx5_indexed_pool *ipool = __atomic_load_n
+				     (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
+
+	if (!ipool) {
+		struct mlx5_indexed_pool *expected = NULL;
+		struct mlx5_indexed_pool_config cfg =
+		    (struct mlx5_indexed_pool_config) {
+		       .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
+								   (index + 1) *
+					   sizeof(struct mlx5_modification_cmd),
+		       .trunk_size = 64,
+		       .grow_trunk = 3,
+		       .grow_shift = 2,
+		       .need_lock = 1,
+		       .release_mem_en = 1,
+		       .malloc = mlx5_malloc,
+		       .free = mlx5_free,
+		       .type = "mlx5_modify_action_resource",
+		};
+
+		cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
+		ipool = mlx5_ipool_create(&cfg);
+		if (!ipool)
+			return NULL;
+		if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
+						 &expected, ipool, false,
+						 __ATOMIC_SEQ_CST,
+						 __ATOMIC_SEQ_CST)) {
+			mlx5_ipool_destroy(ipool);
+			ipool = __atomic_load_n(&sh->mdh_ipools[index],
+						__ATOMIC_SEQ_CST);
+		}
+	}
+	return ipool;
+}
+
 struct mlx5_list_entry *
 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
@@ -5352,12 +5391,20 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
+							  ref->actions_num - 1);
 	int ret;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
+	uint32_t idx;
 
-	entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
-			    SOCKET_ID_ANY);
+	if (unlikely(!ipool)) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL, "cannot allocate modify ipool");
+		return NULL;
+	}
+	entry = mlx5_ipool_zmalloc(ipool, &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5377,25 +5424,29 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 					(sh->ctx, ns, entry,
 					 data_len, &entry->action);
 	if (ret) {
-		mlx5_free(entry);
+		mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create modification action");
 		return NULL;
 	}
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 struct mlx5_list_entry *
-flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
-			struct mlx5_list_entry *oentry, void *cb_ctx)
+flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			void *cb_ctx)
 {
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+	uint32_t idx;
 
-	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
+				  &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5403,14 +5454,18 @@ flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
 		return NULL;
 	}
 	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 void
-flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	mlx5_free(entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *res =
+		container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
@@ -13756,14 +13811,14 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
-			 struct mlx5_list_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
-	mlx5_free(entry);
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v5 21/26] net/mlx5: adjust the hash bucket size

[Suanming Mou]

With the new per core optimization to the list, the hash bucket size
can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index b04e5f8ba6..969d265350 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 640d36c6be..3b5c7840a8 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -376,7 +376,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 906aa43c5a..ca67ce8213 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 45e90bb245..1c5d4c283a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -8881,7 +8881,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8990,8 +8990,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-- 
2.25.1

[dpdk-dev] [PATCH v5 22/26] net/mlx5: enable index pool per-core cache

[Suanming Mou]

This commit enables the tag and header modify action index pool
per-core cache in non-reclaim memory mode.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 3b5c7840a8..f79a094a35 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -217,7 +217,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1129,6 +1130,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e7e4749824..30893c7f38 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1116,6 +1116,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index e39fe43854..739c387c0a 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5361,7 +5361,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
-- 
2.25.1

[dpdk-dev] [PATCH v5 23/26] net/mlx5: optimize hash list table allocate on demand

[Suanming Mou]

Currently, all the hash list tables are allocated during start up.
Since different applications may only use dedicated limited actions,
optimized the hash list table allocate on demand will save initial
memory.

This commit optimizes hash list table allocate on demand.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 969d265350..ec2f8a4aed 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index ca67ce8213..fe86bb40d3 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 739c387c0a..5b8eb096ee 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -311,6 +311,41 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 	}
 }
 
+static inline struct mlx5_hlist *
+flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
+		     const char *name, uint32_t size, bool direct_key,
+		     bool lcores_share, void *ctx,
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove,
+		     mlx5_list_clone_cb cb_clone,
+		     mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_hlist *hl;
+	struct mlx5_hlist *expected = NULL;
+	char s[MLX5_NAME_SIZE];
+
+	hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	if (likely(hl))
+		return hl;
+	snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
+	hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
+			ctx, cb_create, cb_match, cb_remove, cb_clone,
+			cb_clone_free);
+	if (!hl) {
+		DRV_LOG(ERR, "%s hash creation failed", name);
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
+					 __ATOMIC_SEQ_CST,
+					 __ATOMIC_SEQ_CST)) {
+		mlx5_hlist_destroy(hl);
+		hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	}
+	return hl;
+}
+
 /* Update VLAN's VID/PCP based on input rte_flow_action.
  *
  * @param[in] action
@@ -3730,8 +3765,20 @@ flow_dv_encap_decap_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *encaps_decaps;
 	uint64_t key64;
 
+	encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
+				"encaps_decaps",
+				MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
+				true, true, sh,
+				flow_dv_encap_decap_create_cb,
+				flow_dv_encap_decap_match_cb,
+				flow_dv_encap_decap_remove_cb,
+				flow_dv_encap_decap_clone_cb,
+				flow_dv_encap_decap_clone_free_cb);
+	if (unlikely(!encaps_decaps))
+		return -rte_errno;
 	resource->flags = dev_flow->dv.group ? 0 : 1;
 	key64 =  __rte_raw_cksum(&encap_decap_key.v32,
 				 sizeof(encap_decap_key.v32), 0);
@@ -3739,7 +3786,7 @@ flow_dv_encap_decap_resource_register
 	    MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
 	    resource->size)
 		key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
-	entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
+	entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -5745,8 +5792,20 @@ flow_dv_modify_hdr_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *modify_cmds;
 	uint64_t key64;
 
+	modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
+				"hdr_modify",
+				MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
+				true, false, sh,
+				flow_dv_modify_create_cb,
+				flow_dv_modify_match_cb,
+				flow_dv_modify_remove_cb,
+				flow_dv_modify_clone_cb,
+				flow_dv_modify_clone_free_cb);
+	if (unlikely(!modify_cmds))
+		return -rte_errno;
 	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
 								resource->root))
@@ -5754,7 +5813,7 @@ flow_dv_modify_hdr_resource_register
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
 	key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
-	entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
+	entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10537,8 +10596,20 @@ flow_dv_tag_resource_register
 					.error = error,
 					.data = &tag_be24,
 					};
-
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
+	struct mlx5_hlist *tag_table;
+
+	tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
+				      "tags",
+				      MLX5_TAGS_HLIST_ARRAY_SIZE,
+				      false, false, priv->sh,
+				      flow_dv_tag_create_cb,
+				      flow_dv_tag_match_cb,
+				      flow_dv_tag_remove_cb,
+				      flow_dv_tag_clone_cb,
+				      flow_dv_tag_clone_free_cb);
+	if (unlikely(!tag_table))
+		return -rte_errno;
+	entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index a04f93e1d4..5da362a9d5 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
-- 
2.25.1

[dpdk-dev] [PATCH v5 24/26] net/mlx5: change memory release configuration

[Suanming Mou]

This commit changes the index pool memory release configuration
to 0 when memory reclaim mode is not required.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f79a094a35..021a34dd4d 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -798,6 +798,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
-- 
2.25.1

[dpdk-dev] [PATCH v5 25/26] net/mlx5: optimize Rx queue match

[Suanming Mou]

As hrxq struct has the indirect table pointer, while matching the
hrxq, better to use the hrxq indirect table instead of searching
from the list.

This commit optimizes the hrxq indirect table matching.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 7893b3edd4..23685d7654 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v5 26/26] doc: add mlx5 multiple-thread flow insertion optimization

[Suanming Mou]

This commit adds the multiple-thread flow insertion optimization
description.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/nics/mlx5.rst               | 5 +++++
 doc/guides/rel_notes/release_21_08.rst | 1 +
 2 files changed, 6 insertions(+)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 5a3c4f198a..199beb2549 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -448,6 +448,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index 6a902ef9ac..68496496e8 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -88,6 +88,7 @@ New Features
 
   * Added support for meter hierarchy.
   * Added devargs options ``allow_duplicate_pattern``.
+  * Optimize multiple-thread flow insertion rate.
 
 * **Added support for Marvell CNXK crypto driver.**
 
-- 
2.25.1

[dpdk-dev] [PATCH v6 00/26] net/mlx5: insertion rate optimization

[Suanming Mou]

This patch series optimize the flow insertion rate with adding
local cache to index pool and list.

For object which wants efficient index allocate and free, local
cache will be very helpful.

For index pool, two level cache is added, one as local and another
as global. The global cache is able to save all the allocated
index. That means all the allocated index will not be freed. Once
the local cache is full, the extra index will be flushed to the
global cache. Once local cache is empty, first try to fetch more
index from global, if global is still empty, allocate new trunk
and more index.

For list, sub local core list is introduced. The allocated objects
will be added and released only from local list without any locks.
Only the objects need to be shared will be synced from global list.

---

v6: fix compilation issue in "common/mlx5: add list lcore share" patch.

v5: code rebase to the latest version.

v4:
   - split the list utility to commom patch
   - fix one ipool crash issue in mlx5_ipool_get_cache() function.
   - reorganize the common and mlx5 patches.
   - split the doc update to an individual patch.
   - update and rebase to the latest version.

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
  net/mlx5: optimize modify header action memory
  net/mlx5: remove cache term from the list utility
  net/mlx5: add per lcore cache to the list utility
  net/mlx5: minimize list critical sections
  net/mlx5: manage list cache entries release
  net/mlx5: relax the list utility atomic operations
  net/mlx5: allocate list memory by the create API
  common/mlx5: add per-lcore cache to hash list utility
  net/mlx5: move modify header allocator to ipool

Suanming Mou (17):
  net/mlx5: allow limiting the index pool maximum index
  net/mlx5: add indexed pool local cache
  net/mlx5: add index pool foreach define
  net/mlx5: support index pool non-lcore operations
  net/mlx5: replace flow list with index pool
  common/mlx5: move list utility to common
  common/mlx5: add list lcore share
  common/mlx5: call list callbacks with context
  common/mlx5: allocate cache list memory individually
  common/mlx5: optimize cache list object memory
  common/mlx5: support list non-lcore operations
  net/mlx5: adjust the hash bucket size
  net/mlx5: enable index pool per-core cache
  net/mlx5: optimize hash list table allocate on demand
  net/mlx5: change memory release configuration
  net/mlx5: optimize Rx queue match
  doc: add mlx5 multiple-thread flow insertion optimization

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    1 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  294 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  308 +++---
 drivers/net/mlx5/mlx5_flow.h            |  209 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1206 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  627 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2543 insertions(+), 1414 deletions(-)

-- 
2.25.1

[dpdk-dev] [PATCH v6 01/26] net/mlx5: allow limiting the index pool maximum index

[Suanming Mou]

Some ipool instances in the driver are used as ID\index allocator and
added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value.
The ipool will ensure that no index bigger than the maximum value is
provided.

Use this configuration in ID allocator cases instead of the current
logics. This patch add the maximum ID configurable for the index pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 18fe23e4fb..bf2b2ebc72 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index b54517c6df..15870e14c2 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
-- 
2.25.1

[dpdk-dev] [PATCH v6 02/26] net/mlx5: add indexed pool local cache

[Suanming Mou]

For object which wants efficient index allocate and free, local
cache will be very helpful.

Two level cache is introduced to allocate and free the index more
efficient. One as local and the other as global. The global cache
is able to save all the allocated index. That means all the allocated
index will not be freed. Once the local cache is full, the extra
index will be flushed to the global cache. Once local cache is empty,
first try to fetch more index from global, if global is still empty,
allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index bf2b2ebc72..215024632d 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 15870e14c2..0469062695 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -209,6 +209,11 @@ struct mlx5_indexed_pool_config {
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
 	uint32_t max_idx; /* The maximum index can be allocated. */
+	uint32_t per_core_cache;
+	/*
+	 * Cache entry number per core for performance. Should not be
+	 * set with release_mem_en.
+	 */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
@@ -225,14 +230,39 @@ struct mlx5_indexed_trunk {
 	uint8_t data[] __rte_cache_aligned; /* Entry data start. */
 };
 
+struct mlx5_indexed_cache {
+	struct mlx5_indexed_trunk **trunks;
+	volatile uint32_t n_trunk_valid; /* Trunks allocated. */
+	uint32_t n_trunk; /* Trunk pointer array size. */
+	uint32_t ref_cnt;
+	uint32_t len;
+	uint32_t idx[];
+};
+
+struct mlx5_ipool_per_lcore {
+	struct mlx5_indexed_cache *lc;
+	uint32_t len; /**< Current cache count. */
+	uint32_t idx[]; /**< Cache objects. */
+};
+
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
-	rte_spinlock_t lock; /* Pool lock for multiple thread usage. */
-	uint32_t n_trunk_valid; /* Trunks allocated. */
-	uint32_t n_trunk; /* Trunk pointer array size. */
+	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
 	/* Dim of trunk pointer array. */
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t free_list; /* Index to first free trunk. */
+	union {
+		struct {
+			uint32_t n_trunk_valid; /* Trunks allocated. */
+			uint32_t n_trunk; /* Trunk pointer array size. */
+			struct mlx5_indexed_trunk **trunks;
+			uint32_t free_list; /* Index to first free trunk. */
+		};
+		struct {
+			struct mlx5_indexed_cache *gc;
+			/* Global cache. */
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			/* Local cache. */
+		};
+	};
 #ifdef POOL_DEBUG
 	uint32_t n_entry;
 	uint32_t trunk_new;
@@ -542,6 +572,30 @@ int mlx5_ipool_destroy(struct mlx5_indexed_pool *pool);
  */
 void mlx5_ipool_dump(struct mlx5_indexed_pool *pool);
 
+/**
+ * This function flushes all the cache index back to pool trunk.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ *
+ */
+
+void mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool);
+
+/**
+ * This function gets the available entry from pos.
+ *
+ * @param pool
+ *   Pointer to the index memory pool handler.
+ * @param pos
+ *   Pointer to the index position start from.
+ *
+ * @return
+ *  - Pointer to the next available entry.
+ *
+ */
+void *mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos);
+
 /**
  * This function allocates new empty Three-level table.
  *
-- 
2.25.1

[dpdk-dev] [PATCH v6 03/26] net/mlx5: add index pool foreach define

[Suanming Mou]

In some cases, application may want to know all the allocated
index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach
operation.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 96 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 +++
 2 files changed, 104 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 215024632d..32f8d65073 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -529,6 +529,16 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -839,6 +849,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 0469062695..737dd7052d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
-- 
2.25.1

[dpdk-dev] [PATCH v6 04/26] net/mlx5: support index pool non-lcore operations

[Suanming Mou]

This commit supports the index pool non-lcore operations with
an extra cache and lcore lock.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 32f8d65073..f9557c09ff 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -275,6 +275,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->lcore_lock);
 	return pool;
 }
 
@@ -515,20 +516,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -549,15 +544,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -570,29 +577,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -635,6 +653,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -814,7 +847,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -883,7 +916,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 737dd7052d..a509b0a4eb 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t lcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
-- 
2.25.1

[dpdk-dev] [PATCH v6 05/26] net/mlx5: replace flow list with index pool

[Suanming Mou]

The flow list is used to save the create flows and to be used only
when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to
flush all the allocated flows.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 ++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 152 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 127 insertions(+), 112 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index be22d9cbd2..867f450cab 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -832,6 +870,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1575,7 +1614,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1611,6 +1649,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 44fbc2da83..b5c05b12e8 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -325,7 +325,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -793,8 +794,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1529,7 +1532,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index f864c1d701..facab913bf 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1367,7 +1375,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1627,7 +1636,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 2feddb0254..d62361362e 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -3109,31 +3109,6 @@ mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item,
 					 MLX5_ITEM_RANGE_NOT_ACCEPTED, error);
 }
 
-/**
- * Release resource related QUEUE/RSS action split.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param flow
- *   Flow to release id's from.
- */
-static void
-flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
-			     struct rte_flow *flow)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t handle_idx;
-	struct mlx5_flow_handle *dev_handle;
-
-	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
-		       handle_idx, dev_handle, next)
-		if (dev_handle->split_flow_id &&
-		    !dev_handle->is_meter_flow_id)
-			mlx5_ipool_free(priv->sh->ipool
-					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
-					dev_handle->split_flow_id);
-}
-
 static int
 flow_null_validate(struct rte_eth_dev *dev __rte_unused,
 		   const struct rte_flow_attr *attr __rte_unused,
@@ -3429,7 +3404,6 @@ flow_drv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 	const struct mlx5_flow_driver_ops *fops;
 	enum mlx5_flow_drv_type type = flow->drv_type;
 
-	flow_mreg_split_qrss_release(dev, flow);
 	MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
 	fops = flow_get_drv_ops(type);
 	fops->destroy(dev, flow);
@@ -4055,14 +4029,14 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
 
 /* Declare flow create/destroy prototype in advance. */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action actions[],
 		 bool external, struct rte_flow_error *error);
 
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
@@ -4184,8 +4158,8 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	 * be applied, removed, deleted in ardbitrary order
 	 * by list traversing.
 	 */
-	mcp_res->rix_flow = flow_list_create(dev, NULL, &attr, items,
-					 actions, false, error);
+	mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
+					&attr, items, actions, false, error);
 	if (!mcp_res->rix_flow) {
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
 		return NULL;
@@ -4247,7 +4221,7 @@ flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
-	flow_list_destroy(dev, NULL, mcp_res->rix_flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
 }
 
@@ -6187,7 +6161,7 @@ flow_rss_workspace_adjust(struct mlx5_flow_workspace *wks,
  *   A flow index on success, 0 otherwise and rte_errno is set.
  */
 static uint32_t
-flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		 const struct rte_flow_attr *attr,
 		 const struct rte_flow_item items[],
 		 const struct rte_flow_action original_actions[],
@@ -6255,7 +6229,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 				external, hairpin_flow, error);
 	if (ret < 0)
 		goto error_before_hairpin_split;
-	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
+	flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
 	if (!flow) {
 		rte_errno = ENOMEM;
 		goto error_before_hairpin_split;
@@ -6385,12 +6359,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 		if (ret < 0)
 			goto error;
 	}
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
-			     flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
+	flow->type = type;
 	flow_rxq_flags_set(dev, flow);
 	rte_free(translated_actions);
 	tunnel = flow_tunnel_from_rule(wks->flows);
@@ -6412,7 +6381,7 @@ flow_list_create(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_ipool_get
 			(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
 			rss_desc->shared_rss))->refcnt, 1, __ATOMIC_RELAXED);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
+	mlx5_ipool_free(priv->flows[type], idx);
 	rte_errno = ret; /* Restore rte_errno. */
 	ret = rte_errno;
 	rte_errno = ret;
@@ -6464,10 +6433,9 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
 			.type = RTE_FLOW_ACTION_TYPE_END,
 		},
 	};
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow_error error;
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->ctrl_flows,
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
 						   &attr, &pattern,
 						   actions, false, &error);
 }
@@ -6519,8 +6487,6 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		 const struct rte_flow_action actions[],
 		 struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
 	/*
 	 * If the device is not started yet, it is not allowed to created a
 	 * flow from application. PMD default flows and traffic control flows
@@ -6536,8 +6502,9 @@ mlx5_flow_create(struct rte_eth_dev *dev,
 		return NULL;
 	}
 
-	return (void *)(uintptr_t)flow_list_create(dev, &priv->flows,
-				  attr, items, actions, true, error);
+	return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
+						   attr, items, actions,
+						   true, error);
 }
 
 /**
@@ -6545,24 +6512,19 @@ mlx5_flow_create(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list. If this parameter NULL,
- *   there is no flow removal from the list. Be noted that as
- *   flow is add to the indexed list, memory of the indexed
- *   list points to maybe changed as flow destroyed.
  * @param[in] flow_idx
  *   Index of flow to destroy.
  */
 static void
-flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
+flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW], flow_idx);
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
 
 	if (!flow)
 		return;
+	MLX5_ASSERT(flow->type == type);
 	/*
 	 * Update RX queue flags only if port is started, otherwise it is
 	 * already clean.
@@ -6570,12 +6532,6 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
 	flow_drv_destroy(dev, flow);
-	if (list) {
-		rte_spinlock_lock(&priv->flow_list_lock);
-		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
-			     flow_idx, flow, next);
-		rte_spinlock_unlock(&priv->flow_list_lock);
-	}
 	if (flow->tunnel) {
 		struct mlx5_flow_tunnel *tunnel;
 
@@ -6585,7 +6541,7 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
 	flow_mreg_del_copy_action(dev, flow);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
+	mlx5_ipool_free(priv->flows[type], flow_idx);
 }
 
 /**
@@ -6593,18 +6549,21 @@ flow_list_destroy(struct rte_eth_dev *dev, uint32_t *list,
  *
  * @param dev
  *   Pointer to Ethernet device.
- * @param list
- *   Pointer to the Indexed flow list.
+ * @param type
+ *   Flow type to be flushed.
  * @param active
  *   If flushing is called avtively.
  */
 void
-mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active)
+mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+		     bool active)
 {
-	uint32_t num_flushed = 0;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t num_flushed = 0, fidx = 1;
+	struct rte_flow *flow;
 
-	while (*list) {
-		flow_list_destroy(dev, list, *list);
+	MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
+		flow_list_destroy(dev, type, fidx);
 		num_flushed++;
 	}
 	if (active) {
@@ -6776,18 +6735,19 @@ mlx5_flow_pop_thread_workspace(void)
  * @return the number of flows not released.
  */
 int
-mlx5_flow_verify(struct rte_eth_dev *dev)
+mlx5_flow_verify(struct rte_eth_dev *dev __rte_unused)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow *flow;
-	uint32_t idx;
-	int ret = 0;
+	uint32_t idx = 0;
+	int ret = 0, i;
 
-	ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], priv->flows, idx,
-		      flow, next) {
-		DRV_LOG(DEBUG, "port %u flow %p still referenced",
-			dev->data->port_id, (void *)flow);
-		++ret;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		MLX5_IPOOL_FOREACH(priv->flows[i], idx, flow) {
+			DRV_LOG(DEBUG, "port %u flow %p still referenced",
+				dev->data->port_id, (void *)flow);
+			ret++;
+		}
 	}
 	return ret;
 }
@@ -6807,7 +6767,6 @@ int
 mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 			    uint32_t queue)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_attr attr = {
 		.egress = 1,
 		.priority = 0,
@@ -6840,8 +6799,8 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
 	actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
 	actions[0].conf = &jump;
 	actions[1].type = RTE_FLOW_ACTION_TYPE_END;
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx) {
 		DRV_LOG(DEBUG,
 			"Failed to create ctrl flow: rte_errno(%d),"
@@ -6930,8 +6889,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
 		action_rss.types = 0;
 	for (i = 0; i != priv->reta_idx_n; ++i)
 		queue[i] = (*priv->reta_idx)[i];
-	flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+				    &attr, items, actions, false, &error);
 	if (!flow_idx)
 		return -rte_errno;
 	return 0;
@@ -6972,7 +6931,6 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev,
 int
 mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	/*
 	 * The LACP matching is done by only using ether type since using
 	 * a multicast dst mac causes kernel to give low priority to this flow.
@@ -7006,8 +6964,9 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
 		},
 	};
 	struct rte_flow_error error;
-	uint32_t flow_idx = flow_list_create(dev, &priv->ctrl_flows,
-				&attr, items, actions, false, &error);
+	uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+					&attr, items, actions,
+					false, &error);
 
 	if (!flow_idx)
 		return -rte_errno;
@@ -7025,9 +6984,8 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
 		  struct rte_flow *flow,
 		  struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	flow_list_destroy(dev, &priv->flows, (uintptr_t)(void *)flow);
+	flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+				(uintptr_t)(void *)flow);
 	return 0;
 }
 
@@ -7041,9 +6999,7 @@ int
 mlx5_flow_flush(struct rte_eth_dev *dev,
 		struct rte_flow_error *error __rte_unused)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, false);
 	return 0;
 }
 
@@ -7094,8 +7050,7 @@ flow_drv_query(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct mlx5_flow_driver_ops *fops;
-	struct rte_flow *flow = mlx5_ipool_get(priv->sh->ipool
-					       [MLX5_IPOOL_RTE_FLOW],
+	struct rte_flow *flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 					       flow_idx);
 	enum mlx5_flow_drv_type ftype;
 
@@ -7961,14 +7916,14 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
 		if (!config->dv_flow_en)
 			break;
 		/* Create internal flow, validation skips copy action. */
-		flow_idx = flow_list_create(dev, NULL, &attr, items,
-					    actions, false, &error);
-		flow = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
+		flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
+					items, actions, false, &error);
+		flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
 				      flow_idx);
 		if (!flow)
 			continue;
 		config->flow_mreg_c[n++] = idx;
-		flow_list_destroy(dev, NULL, flow_idx);
+		flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
 	}
 	for (; n < MLX5_MREG_C_NUM; ++n)
 		config->flow_mreg_c[n] = REG_NON;
@@ -8163,8 +8118,7 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
 	/* dump all */
 	if (!flow_idx) {
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-		ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW],
-						priv->flows, idx, flow, next)
+		MLX5_IPOOL_FOREACH(priv->flows[MLX5_FLOW_TYPE_GEN], idx, flow)
 			mlx5_flow_dev_dump_ipool(dev, flow, file, error);
 #endif
 		return mlx5_devx_cmd_flow_dump(sh->fdb_domain,
@@ -8172,8 +8126,8 @@ mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow_idx,
 					sh->tx_domain, file);
 	}
 	/* dump one */
-	flow = mlx5_ipool_get(priv->sh->ipool
-			[MLX5_IPOOL_RTE_FLOW], (uintptr_t)(void *)flow_idx);
+	flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
+			(uintptr_t)(void *)flow_idx);
 	if (!flow)
 		return -ENOENT;
 
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 7d97c5880f..adadcfa425 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -998,9 +998,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 2f4c0eeb5b..49bc85f7b3 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13901,6 +13901,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index ae7fcca229..7cb8920d6b 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index e30b682822..17716b66c9 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
-- 
2.25.1

[dpdk-dev] [PATCH v6 06/26] net/mlx5: optimize modify header action memory

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Define the types of the modify header action fields to be with the
minimum size needed for the optional values range.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_glue.h b/drivers/common/mlx5/linux/mlx5_glue.h
index 61f40d5478..f39ef2dac7 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.h
+++ b/drivers/common/mlx5/linux/mlx5_glue.h
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
diff --git a/drivers/net/mlx5/linux/mlx5_flow_os.h b/drivers/net/mlx5/linux/mlx5_flow_os.h
index cee685015b..1926d26410 100644
--- a/drivers/net/mlx5/linux/mlx5_flow_os.h
+++ b/drivers/net/mlx5/linux/mlx5_flow_os.h
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index adadcfa425..f1ebc153f2 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -524,11 +524,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 49bc85f7b3..0f8b916991 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5006,21 +5006,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5622,10 +5622,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
-- 
2.25.1

[dpdk-dev] [PATCH v6 07/26] net/mlx5: remove cache term from the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The internal mlx5 list tool is used mainly when the list objects need to
be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread
cache management.

To prevent confusing, remove the current "cache" term from the API's
names.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 561 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 456 insertions(+), 461 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 867f450cab..5d537fa3e4 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -509,8 +509,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1713,7 +1713,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1774,7 +1774,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index b5c05b12e8..5cd2cfb07c 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1612,7 +1612,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index facab913bf..de5cd2e06e 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1137,10 +1137,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1244,7 +1244,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1382,7 +1382,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1392,7 +1392,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index f1ebc153f2..cbae4f1872 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -468,7 +468,7 @@ struct mlx5_flow_dv_match_params {
 
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
-	struct mlx5_cache_entry entry; /**< Pointer to the next element. */
+	struct mlx5_list_entry entry; /**< Pointer to the next element. */
 	struct mlx5_flow_tbl_resource *tbl;
 	/**< Pointer to the table(group) the matcher associated with. */
 	void *matcher_object; /**< Pointer to DV matcher */
@@ -548,7 +548,7 @@ struct mlx5_flow_dv_jump_tbl_resource {
 
 /* Port ID resource structure. */
 struct mlx5_flow_dv_port_id_action_resource {
-	struct mlx5_cache_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Action object. */
 	uint32_t port_id; /**< Port ID value. */
 	uint32_t idx; /**< Indexed pool memory index. */
@@ -556,7 +556,7 @@ struct mlx5_flow_dv_port_id_action_resource {
 
 /* Push VLAN action resource structure */
 struct mlx5_flow_dv_push_vlan_action_resource {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* Cache entry. */
 	void *action; /**< Action object. */
 	uint8_t ft_type; /**< Flow table type, Rx, Tx or FDB. */
 	rte_be32_t vlan_tag; /**< VLAN tag value. */
@@ -591,7 +591,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_cache_list matchers;
+	struct mlx5_list matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
@@ -632,7 +632,7 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	union {
 		void *verbs_action; /**< Verbs sample action object. */
 		void **sub_actions; /**< Sample sub-action array. */
@@ -654,7 +654,7 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	struct mlx5_cache_entry entry; /**< Cache entry. */
+	struct mlx5_list_entry entry; /**< Cache entry. */
 	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
@@ -1631,43 +1631,45 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
 
-int flow_dv_matcher_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *ctx);
-struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_port_id_match_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry);
-
-int flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list,
-			       struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_sample_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
-			      struct mlx5_cache_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
-				struct mlx5_cache_entry *entry, void *cb_ctx);
-struct mlx5_cache_entry *flow_dv_dest_array_create_cb
-				(struct mlx5_cache_list *list,
-				 struct mlx5_cache_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry);
+int flow_dv_matcher_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *ctx);
+void flow_dv_matcher_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry);
+
+int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
+			       struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
+						 struct mlx5_list_entry *entry,
+						 void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry, void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 0f8b916991..3aa3192dd7 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3607,18 +3607,17 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 			     uint64_t key __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
-
-	cache_resource = container_of(entry,
-				      struct mlx5_flow_dv_encap_decap_resource,
-				      entry);
-	if (resource->reformat_type == cache_resource->reformat_type &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->flags == cache_resource->flags &&
-	    resource->size == cache_resource->size &&
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
+
+	resource = container_of(entry, struct mlx5_flow_dv_encap_decap_resource,
+				entry);
+	if (resource->reformat_type == ctx_resource->reformat_type &&
+	    resource->ft_type == ctx_resource->ft_type &&
+	    resource->flags == ctx_resource->flags &&
+	    resource->size == ctx_resource->size &&
 	    !memcmp((const void *)resource->buf,
-		    (const void *)cache_resource->buf,
+		    (const void *)ctx_resource->buf,
 		    resource->size))
 		return 0;
 	return -1;
@@ -3645,31 +3644,30 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
-	struct mlx5_flow_dv_encap_decap_resource *resource = ctx->data;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 	uint32_t idx;
 	int ret;
 
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
 	/* Register new encap/decap resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-				       &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
-	cache_resource->idx = idx;
-	ret = mlx5_flow_os_create_flow_action_packet_reformat
-					(sh->ctx, domain, cache_resource,
-					 &cache_resource->action);
+	*resource = *ctx_resource;
+	resource->idx = idx;
+	ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->ctx, domain,
+							      resource,
+							     &resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3678,7 +3676,7 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 		return NULL;
 	}
 
-	return &cache_resource->entry;
+	return &resource->entry;
 }
 
 /**
@@ -3782,8 +3780,8 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
@@ -3793,30 +3791,30 @@ flow_dv_port_id_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->port_id != res->port_id;
 }
 
-struct mlx5_cache_entry *
-flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_port_id_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate port_id action cache memory");
+				   "cannot allocate port_id action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	ret = mlx5_flow_os_create_flow_action_dest_port(sh->fdb_domain,
 							ref->port_id,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3824,8 +3822,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 				   "cannot create action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3833,8 +3831,8 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3846,30 +3844,30 @@ flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_port_id_action_resource_register
 			(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_port_id_action_resource *resource,
+			 struct mlx5_flow_dv_port_id_action_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->dv.port_id_action = cache;
-	dev_flow->handle->rix_port_id_action = cache->idx;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->dv.port_id_action = resource;
+	dev_flow->handle->rix_port_id_action = resource->idx;
 	return 0;
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
@@ -3879,28 +3877,28 @@ flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list __rte_unused,
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
-struct mlx5_cache_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_push_vlan_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	struct mlx5dv_dr_domain *domain;
 	uint32_t idx;
 	int ret;
 
 	/* Register new port id action resource. */
-	cache = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
-	if (!cache) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot allocate push_vlan action cache memory");
+				   "cannot allocate push_vlan action memory");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (ref->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
@@ -3908,7 +3906,7 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 	else
 		domain = sh->tx_domain;
 	ret = mlx5_flow_os_create_flow_action_push_vlan(domain, ref->vlan_tag,
-							&cache->action);
+							&resource->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
@@ -3916,8 +3914,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 				   "cannot create push vlan action");
 		return NULL;
 	}
-	cache->idx = idx;
-	return &cache->entry;
+	resource->idx = idx;
+	return &resource->entry;
 }
 
 /**
@@ -3925,8 +3923,8 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
  *
  * @param [in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- *   Pointer to port ID action resource.
+ * @param[in, out] ref
+ *   Pointer to port ID action resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -3938,25 +3936,25 @@ flow_dv_push_vlan_create_cb(struct mlx5_cache_list *list,
 static int
 flow_dv_push_vlan_action_resource_register
 		       (struct rte_eth_dev *dev,
-			struct mlx5_flow_dv_push_vlan_action_resource *resource,
+			struct mlx5_flow_dv_push_vlan_action_resource *ref,
 			struct mlx5_flow *dev_flow,
 			struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache = container_of(entry, typeof(*cache), entry);
+	resource = container_of(entry, typeof(*resource), entry);
 
-	dev_flow->handle->dvh.rix_push_vlan = cache->idx;
-	dev_flow->dv.push_vlan_res = cache;
+	dev_flow->handle->dvh.rix_push_vlan = resource->idx;
+	dev_flow->dv.push_vlan_res = resource;
 	return 0;
 }
 
@@ -9957,13 +9955,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 			return NULL;
 		}
 	}
-	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_cache",
+	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_cache_list_init(&tbl_data->matchers, matcher_name, 0, sh,
-			     flow_dv_matcher_create_cb,
-			     flow_dv_matcher_match_cb,
-			     flow_dv_matcher_remove_cb);
+	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+			 flow_dv_matcher_create_cb,
+			 flow_dv_matcher_match_cb,
+			 flow_dv_matcher_remove_cb);
 	return &tbl_data->entry;
 }
 
@@ -10091,7 +10089,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_cache_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(&tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10119,8 +10117,8 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
@@ -10133,15 +10131,15 @@ flow_dv_matcher_match_cb(struct mlx5_cache_list *list __rte_unused,
 		      (const void *)ref->mask.buf, ref->mask.size);
 }
 
-struct mlx5_cache_entry *
-flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_matcher_create_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
 			  void *cb_ctx)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5dv_flow_matcher_attr dv_attr = {
 		.type = IBV_FLOW_ATTR_NORMAL,
 		.match_mask = (void *)&ref->mask,
@@ -10150,29 +10148,30 @@ flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
 							    typeof(*tbl), tbl);
 	int ret;
 
-	cache = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*cache), 0, SOCKET_ID_ANY);
-	if (!cache) {
+	resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
+			       SOCKET_ID_ANY);
+	if (!resource) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	*cache = *ref;
+	*resource = *ref;
 	dv_attr.match_criteria_enable =
-		flow_dv_matcher_enable(cache->mask.buf);
+		flow_dv_matcher_enable(resource->mask.buf);
 	dv_attr.priority = ref->priority;
 	if (tbl->is_egress)
 		dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
 	ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, tbl->tbl.obj,
-					       &cache->matcher_object);
+					       &resource->matcher_object);
 	if (ret) {
-		mlx5_free(cache);
+		mlx5_free(resource);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot create matcher");
 		return NULL;
 	}
-	return &cache->entry;
+	return &resource->entry;
 }
 
 /**
@@ -10201,8 +10200,8 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 			 uint32_t group_id,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
-	struct mlx5_flow_dv_matcher *cache;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_dv_matcher *resource;
 	struct mlx5_flow_tbl_resource *tbl;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 	struct mlx5_flow_cb_ctx ctx = {
@@ -10222,15 +10221,15 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 					  "cannot allocate ref memory");
 	}
-	cache = container_of(entry, typeof(*cache), entry);
-	dev_flow->handle->dvh.matcher = cache;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.matcher = resource;
 	return 0;
 }
 
@@ -10298,15 +10297,15 @@ flow_dv_tag_resource_register
 			 struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_tag_resource *cache_resource;
+	struct mlx5_flow_dv_tag_resource *resource;
 	struct mlx5_hlist_entry *entry;
 
 	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
 	if (entry) {
-		cache_resource = container_of
-			(entry, struct mlx5_flow_dv_tag_resource, entry);
-		dev_flow->handle->dvh.rix_tag = cache_resource->idx;
-		dev_flow->dv.tag_resource = cache_resource;
+		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
+					entry);
+		dev_flow->handle->dvh.rix_tag = resource->idx;
+		dev_flow->dv.tag_resource = resource;
 		return 0;
 	}
 	return -rte_errno;
@@ -10633,68 +10632,69 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_cache_list *list __rte_unused,
-			struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-
-	if (resource->ratio == cache_resource->ratio &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    resource->ft_id == cache_resource->ft_id &&
-	    resource->set_action == cache_resource->set_action &&
-	    !memcmp((void *)&resource->sample_act,
-		    (void *)&cache_resource->sample_act,
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+
+	if (ctx_resource->ratio == resource->ratio &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    ctx_resource->ft_id == resource->ft_id &&
+	    ctx_resource->set_action == resource->set_action &&
+	    !memcmp((void *)&ctx_resource->sample_act,
+		    (void *)&resource->sample_act,
 		    sizeof(struct mlx5_flow_sub_actions_list))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
 		flow_dv_sample_sub_actions_release(dev,
-						&resource->sample_idx);
+						   &ctx_resource->sample_idx);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_sample_resource *resource = ctx->data;
-	void **sample_dv_actions = resource->sub_actions;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *ctx_resource = ctx->data;
+	void **sample_dv_actions = ctx_resource->sub_actions;
+	struct mlx5_flow_dv_sample_resource *resource;
 	struct mlx5dv_dr_flow_sampler_attr sampler_attr;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	struct mlx5_flow_tbl_resource *tbl;
 	uint32_t idx = 0;
 	const uint32_t next_ft_step = 1;
-	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
+	uint32_t next_ft_id = ctx_resource->ft_id + next_ft_step;
 	uint8_t is_egress = 0;
 	uint8_t is_transfer = 0;
 	struct rte_flow_error *error = ctx->error;
 
 	/* Register new sample resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
-	if (!cache_resource) {
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	/* Create normal path table level */
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		is_transfer = 1;
-	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+	else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
 		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
 					is_egress, is_transfer,
@@ -10707,8 +10707,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 					  "for sample");
 		goto error;
 	}
-	cache_resource->normal_path_tbl = tbl;
-	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
+	resource->normal_path_tbl = tbl;
+	if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
 		if (!sh->default_miss_action) {
 			rte_flow_error_set(error, ENOMEM,
 						RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10717,33 +10717,33 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
 						"created");
 			goto error;
 		}
-		sample_dv_actions[resource->sample_act.actions_num++] =
+		sample_dv_actions[ctx_resource->sample_act.actions_num++] =
 						sh->default_miss_action;
 	}
 	/* Create a DR sample action */
-	sampler_attr.sample_ratio = cache_resource->ratio;
+	sampler_attr.sample_ratio = resource->ratio;
 	sampler_attr.default_next_table = tbl->obj;
-	sampler_attr.num_sample_actions = resource->sample_act.actions_num;
+	sampler_attr.num_sample_actions = ctx_resource->sample_act.actions_num;
 	sampler_attr.sample_actions = (struct mlx5dv_dr_action **)
 							&sample_dv_actions[0];
-	sampler_attr.action = cache_resource->set_action;
+	sampler_attr.action = resource->set_action;
 	if (mlx5_os_flow_dr_create_flow_action_sampler
-			(&sampler_attr, &cache_resource->verbs_action)) {
+			(&sampler_attr, &resource->verbs_action)) {
 		rte_flow_error_set(error, ENOMEM,
 					RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					NULL, "cannot create sample action");
 		goto error;
 	}
-	cache_resource->idx = idx;
-	cache_resource->dev = dev;
-	return &cache_resource->entry;
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
 error:
-	if (cache_resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
+	if (resource->ft_type != MLX5DV_FLOW_TABLE_TYPE_FDB)
 		flow_dv_sample_sub_actions_release(dev,
-						   &cache_resource->sample_idx);
-	if (cache_resource->normal_path_tbl)
+						   &resource->sample_idx);
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-				cache_resource->normal_path_tbl);
+				resource->normal_path_tbl);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_SAMPLE], idx);
 	return NULL;
 
@@ -10754,8 +10754,8 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to sample resource.
+ * @param[in] ref
+ *   Pointer to sample resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10766,66 +10766,66 @@ flow_dv_sample_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_sample_resource *resource,
+			 struct mlx5_flow_dv_sample_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_sample = cache_resource->idx;
-	dev_flow->dv.sample_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_sample = resource->idx;
+	dev_flow->dv.sample_res = resource;
 	return 0;
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_cache_list *list __rte_unused,
-			    struct mlx5_cache_entry *entry, void *cb_ctx)
+flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
-	if (resource->num_of_dest == cache_resource->num_of_dest &&
-	    resource->ft_type == cache_resource->ft_type &&
-	    !memcmp((void *)cache_resource->sample_act,
-		    (void *)resource->sample_act,
-		   (resource->num_of_dest *
+	if (ctx_resource->num_of_dest == resource->num_of_dest &&
+	    ctx_resource->ft_type == resource->ft_type &&
+	    !memcmp((void *)resource->sample_act,
+		    (void *)ctx_resource->sample_act,
+		   (ctx_resource->num_of_dest *
 		   sizeof(struct mlx5_flow_sub_actions_list)))) {
 		/*
 		 * Existing sample action should release the prepared
 		 * sub-actions reference counter.
 		 */
-		for (idx = 0; idx < resource->num_of_dest; idx++)
+		for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 			flow_dv_sample_sub_actions_release(dev,
-					&resource->sample_idx[idx]);
+					&ctx_resource->sample_idx[idx]);
 		return 0;
 	}
 	return 1;
 }
 
-struct mlx5_cache_entry *
-flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	struct mlx5_flow_dv_dest_array_resource *resource = ctx->data;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct mlx5dv_dr_action_dest_attr *dest_attr[MLX5_MAX_DEST_NUM] = { 0 };
 	struct mlx5dv_dr_action_dest_reformat dest_reformat[MLX5_MAX_DEST_NUM];
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -10838,23 +10838,23 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	int ret;
 
 	/* Register new destination array resource. */
-	cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
 					    &res_idx);
-	if (!cache_resource) {
+	if (!resource) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 					  NULL,
 					  "cannot allocate resource memory");
 		return NULL;
 	}
-	*cache_resource = *resource;
+	*resource = *ctx_resource;
 	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
 	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		dest_attr[idx] = (struct mlx5dv_dr_action_dest_attr *)
 				 mlx5_malloc(MLX5_MEM_ZERO,
 				 sizeof(struct mlx5dv_dr_action_dest_attr),
@@ -10867,7 +10867,7 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 			goto error;
 		}
 		dest_attr[idx]->type = MLX5DV_DR_ACTION_DEST;
-		sample_act = &resource->sample_act[idx];
+		sample_act = &ctx_resource->sample_act[idx];
 		action_flags = sample_act->action_flags;
 		switch (action_flags) {
 		case MLX5_FLOW_ACTION_QUEUE:
@@ -10898,9 +10898,9 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 	/* create a dest array actioin */
 	ret = mlx5_os_flow_dr_create_flow_action_dest_array
 						(domain,
-						 cache_resource->num_of_dest,
+						 resource->num_of_dest,
 						 dest_attr,
-						 &cache_resource->action);
+						 &resource->action);
 	if (ret) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -10908,19 +10908,18 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
 				   "cannot create destination array action");
 		goto error;
 	}
-	cache_resource->idx = res_idx;
-	cache_resource->dev = dev;
-	for (idx = 0; idx < resource->num_of_dest; idx++)
+	resource->idx = res_idx;
+	resource->dev = dev;
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++)
 		mlx5_free(dest_attr[idx]);
-	return &cache_resource->entry;
+	return &resource->entry;
 error:
-	for (idx = 0; idx < resource->num_of_dest; idx++) {
+	for (idx = 0; idx < ctx_resource->num_of_dest; idx++) {
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[idx]);
+						   &resource->sample_idx[idx]);
 		if (dest_attr[idx])
 			mlx5_free(dest_attr[idx]);
 	}
-
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DEST_ARRAY], res_idx);
 	return NULL;
 }
@@ -10930,8 +10929,8 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] resource
- *   Pointer to destination array resource.
+ * @param[in] ref
+ *   Pointer to destination array resource reference.
  * @parm[in, out] dev_flow
  *   Pointer to the dev_flow.
  * @param[out] error
@@ -10942,25 +10941,25 @@ flow_dv_dest_array_create_cb(struct mlx5_cache_list *list __rte_unused,
  */
 static int
 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 struct mlx5_flow_dv_dest_array_resource *resource,
+			 struct mlx5_flow_dv_dest_array_resource *ref,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = resource,
+		.data = ref,
 	};
 
-	entry = mlx5_cache_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
-	cache_resource = container_of(entry, typeof(*cache_resource), entry);
-	dev_flow->handle->dvh.rix_dest_array = cache_resource->idx;
-	dev_flow->dv.dest_array_res = cache_resource;
+	resource = container_of(entry, typeof(*resource), entry);
+	dev_flow->handle->dvh.rix_dest_array = resource->idx;
+	dev_flow->dv.dest_array_res = resource;
 	return 0;
 }
 
@@ -13402,14 +13401,15 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			  struct mlx5_cache_entry *entry)
+flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+			  struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_matcher *cache = container_of(entry, typeof(*cache),
-							  entry);
+	struct mlx5_flow_dv_matcher *resource = container_of(entry,
+							     typeof(*resource),
+							     entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_matcher(cache->matcher_object));
-	mlx5_free(cache);
+	claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
+	mlx5_free(resource);
 }
 
 /**
@@ -13433,7 +13433,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_cache_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13452,7 +13452,7 @@ flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
-		container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
@@ -13474,15 +13474,14 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
 				     uint32_t encap_decap_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	struct mlx5_flow_dv_encap_decap_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-					encap_decap_idx);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+				  encap_decap_idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->action);
-	return mlx5_hlist_unregister(priv->sh->encaps_decaps,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_hlist_unregister(priv->sh->encaps_decaps, &resource->entry);
 }
 
 /**
@@ -13544,15 +13543,15 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry)
+flow_dv_port_id_remove_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_port_id_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+				  container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
 
 /**
@@ -13571,14 +13570,14 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 					uint32_t port_id)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_port_id_action_resource *cache;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PORT_ID], port_id);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->port_id_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13601,15 +13600,15 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
-			    struct mlx5_cache_entry *entry)
+flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
 {
 	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache =
-			container_of(entry, typeof(*cache), entry);
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
 
-	claim_zero(mlx5_flow_os_destroy_flow_action(cache->action));
-	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], cache->idx);
+	claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
 
 /**
@@ -13628,15 +13627,15 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 					  struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_push_vlan_action_resource *cache;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx = handle->dvh.rix_push_vlan;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN], idx);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->push_vlan_action_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+				    &resource->entry);
 }
 
 /**
@@ -13673,26 +13672,24 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			 struct mlx5_cache_entry *entry)
+flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_sample_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
+							      typeof(*resource),
+							      entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	if (cache_resource->verbs_action)
+	if (resource->verbs_action)
 		claim_zero(mlx5_flow_os_destroy_flow_action
-				(cache_resource->verbs_action));
-	if (cache_resource->normal_path_tbl)
+						      (resource->verbs_action));
+	if (resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
-			cache_resource->normal_path_tbl);
-	flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			cache_resource->idx);
-	DRV_LOG(DEBUG, "sample resource %p: removed",
-		(void *)cache_resource);
+					     resource->normal_path_tbl);
+	flow_dv_sample_sub_actions_release(dev, &resource->sample_idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
+	DRV_LOG(DEBUG, "sample resource %p: removed", (void *)resource);
 }
 
 /**
@@ -13711,38 +13708,36 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 				     struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_sample_resource *cache_resource;
+	struct mlx5_flow_dv_sample_resource *resource;
 
-	cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			 handle->dvh.rix_sample);
-	if (!cache_resource)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+				  handle->dvh.rix_sample);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache_resource->verbs_action);
-	return mlx5_cache_unregister(&priv->sh->sample_action_list,
-				     &cache_resource->entry);
+	MLX5_ASSERT(resource->verbs_action);
+	return mlx5_list_unregister(&priv->sh->sample_action_list,
+				    &resource->entry);
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
-	struct mlx5_flow_dv_dest_array_resource *cache_resource =
-			container_of(entry, typeof(*cache_resource), entry);
-	struct rte_eth_dev *dev = cache_resource->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	uint32_t i = 0;
 
-	MLX5_ASSERT(cache_resource->action);
-	if (cache_resource->action)
-		claim_zero(mlx5_flow_os_destroy_flow_action
-					(cache_resource->action));
-	for (; i < cache_resource->num_of_dest; i++)
+	MLX5_ASSERT(resource->action);
+	if (resource->action)
+		claim_zero(mlx5_flow_os_destroy_flow_action(resource->action));
+	for (; i < resource->num_of_dest; i++)
 		flow_dv_sample_sub_actions_release(dev,
-				&cache_resource->sample_idx[i]);
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			cache_resource->idx);
+						   &resource->sample_idx[i]);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
 	DRV_LOG(DEBUG, "destination array resource %p: removed",
-		(void *)cache_resource);
+		(void *)resource);
 }
 
 /**
@@ -13761,15 +13756,15 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 				    struct mlx5_flow_handle *handle)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_dv_dest_array_resource *cache;
+	struct mlx5_flow_dv_dest_array_resource *resource;
 
-	cache = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
-			       handle->dvh.rix_dest_array);
-	if (!cache)
+	resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				  handle->dvh.rix_dest_array);
+	if (!resource)
 		return 0;
-	MLX5_ASSERT(cache->action);
-	return mlx5_cache_unregister(&priv->sh->dest_array_list,
-				     &cache->entry);
+	MLX5_ASSERT(resource->action);
+	return mlx5_list_unregister(&priv->sh->dest_array_list,
+				    &resource->entry);
 }
 
 static void
@@ -14619,7 +14614,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 			claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
 			tbl = container_of(color_rule->matcher->tbl,
 					typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 						&color_rule->matcher->entry);
 			TAILQ_REMOVE(&sub_policy->color_rules[i],
 					color_rule, next_port);
@@ -15412,8 +15407,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_cache_unregister(&tbl->matchers,
-				      &mtrmng->def_matcher[i]->entry);
+			mlx5_list_unregister(&tbl->matchers,
+					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
 		for (j = 0; j < MLX5_REG_BITS; j++) {
@@ -15422,8 +15417,8 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_cache_unregister(&tbl->matchers,
-					&mtrmng->drop_matcher[i][j]->entry);
+				mlx5_list_unregister(&tbl->matchers,
+					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
 		}
@@ -15521,7 +15516,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 			struct mlx5_flow_dv_matcher **policy_matcher,
 			struct rte_flow_error *error)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_resource *tbl_rsc = sub_policy->tbl_rsc;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
@@ -15557,7 +15552,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -15665,7 +15660,7 @@ __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
 			struct mlx5_flow_tbl_data_entry *tbl =
 				container_of(color_rule->matcher->tbl,
 						typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 						&color_rule->matcher->entry);
 		}
 		mlx5_free(color_rule);
@@ -16017,7 +16012,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 						     0, &error);
 	uint32_t mtr_id_mask = (UINT32_C(1) << mtrmng->max_mtr_bits) - 1;
 	uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_dv_matcher matcher = {
 		.mask = {
 			.size = sizeof(matcher.mask.buf) -
@@ -16063,7 +16058,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -16100,7 +16095,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
@@ -16517,7 +16512,7 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
 			struct mlx5_flow_tbl_data_entry *tbl =
 				container_of(color_rule->matcher->tbl,
 						typeof(*tbl), tbl);
-			mlx5_cache_unregister(&tbl->matchers,
+			mlx5_list_unregister(&tbl->matchers,
 						&color_rule->matcher->entry);
 		}
 		mlx5_free(color_rule);
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 1b264e5994..3dcc71d51d 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index bb9a908087..8395332507 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index f9557c09ff..4536ca807d 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,29 +9,29 @@
 #include "mlx5_utils.h"
 
 
-/********************* Cache list ************************/
+/********************* MLX5 list ************************/
 
-static struct mlx5_cache_entry *
-mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
-			     struct mlx5_cache_entry *entry __rte_unused,
+static struct mlx5_list_entry *
+mlx5_list_default_create_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry __rte_unused,
 			     void *ctx __rte_unused)
 {
 	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
 }
 
 static void
-mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
-			     struct mlx5_cache_entry *entry)
+mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
 }
 
 int
-mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
+mlx5_list_create(struct mlx5_list *list, const char *name,
 		     uint32_t entry_size, void *ctx,
-		     mlx5_cache_create_cb cb_create,
-		     mlx5_cache_match_cb cb_match,
-		     mlx5_cache_remove_cb cb_remove)
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove)
 {
 	MLX5_ASSERT(list);
 	if (!cb_match || (!cb_create ^ !cb_remove))
@@ -40,19 +40,19 @@ mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
+	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
+	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
 	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	LIST_INIT(&list->head);
 	return 0;
 }
 
-static struct mlx5_cache_entry *
-__cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	LIST_FOREACH(entry, &list->head, next) {
 		if (list->cb_match(list, entry, ctx))
@@ -60,7 +60,7 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 		if (reuse) {
 			__atomic_add_fetch(&entry->ref_cnt, 1,
 					   __ATOMIC_RELAXED);
-			DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
 		break;
@@ -68,33 +68,33 @@ __cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_cache_entry *
-cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+static struct mlx5_list_entry *
+list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __cache_lookup(list, ctx, reuse);
+	entry = __list_lookup(list, ctx, reuse);
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	return cache_lookup(list, ctx, false);
+	return list_lookup(list, ctx, false);
 }
 
-struct mlx5_cache_entry *
-mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	uint32_t prev_gen_cnt = 0;
 
 	MLX5_ASSERT(list);
 	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
 	/* Lookup with read lock, reuse if found. */
-	entry = cache_lookup(list, ctx, true);
+	entry = list_lookup(list, ctx, true);
 	if (entry)
 		return entry;
 	/* Not found, append with write lock - block read from other threads. */
@@ -102,13 +102,13 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __cache_lookup(list, ctx, true);
+		entry = __list_lookup(list, ctx, true);
 		if (entry)
 			goto done;
 	}
 	entry = list->cb_create(list, entry, ctx);
 	if (!entry) {
-		DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
+		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
 			list->name, (void *)entry);
 		goto done;
 	}
@@ -116,7 +116,7 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 	LIST_INSERT_HEAD(&list->head, entry, next);
 	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 done:
 	rte_rwlock_write_unlock(&list->lock);
@@ -124,12 +124,12 @@ mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
 }
 
 int
-mlx5_cache_unregister(struct mlx5_cache_list *list,
-		      struct mlx5_cache_entry *entry)
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
 {
 	rte_rwlock_write_lock(&list->lock);
 	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
 		list->name, (void *)entry, entry->ref_cnt);
 	if (--entry->ref_cnt) {
 		rte_rwlock_write_unlock(&list->lock);
@@ -140,15 +140,15 @@ mlx5_cache_unregister(struct mlx5_cache_list *list,
 	LIST_REMOVE(entry, next);
 	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 		list->name, (void *)entry);
 	return 0;
 }
 
 void
-mlx5_cache_list_destroy(struct mlx5_cache_list *list)
+mlx5_list_destroy(struct mlx5_list *list)
 {
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 
 	MLX5_ASSERT(list);
 	/* no LIST_FOREACH_SAFE, using while instead */
@@ -156,14 +156,14 @@ mlx5_cache_list_destroy(struct mlx5_cache_list *list)
 		entry = LIST_FIRST(&list->head);
 		LIST_REMOVE(entry, next);
 		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
 			list->name, (void *)entry);
 	}
 	memset(list, 0, sizeof(*list));
 }
 
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
+mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
 	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index a509b0a4eb..cfb3cb6180 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -297,19 +297,19 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ cache list *****************************/
+/************************ mlx5 list *****************************/
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
 
-struct mlx5_cache_list;
+struct mlx5_list;
 
 /**
- * Structure of the entry in the cache list, user should define its own struct
+ * Structure of the entry in the mlx5 list, user should define its own struct
  * that contains this in order to store the data.
  */
-struct mlx5_cache_entry {
-	LIST_ENTRY(mlx5_cache_entry) next; /* Entry pointers in the list. */
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* Reference count. */
 };
 
@@ -317,18 +317,18 @@ struct mlx5_cache_entry {
  * Type of callback function for entry removal.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
-				     struct mlx5_cache_entry *entry);
+typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
+				     struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -337,14 +337,14 @@ typedef void (*mlx5_cache_remove_cb)(struct mlx5_cache_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
-				   struct mlx5_cache_entry *entry, void *ctx);
+typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+				   struct mlx5_list_entry *entry, void *ctx);
 
 /**
- * Type of function for user defined cache list entry creation.
+ * Type of function for user defined mlx5 list entry creation.
  *
  * @param list
- *   The cache list.
+ *   The mlx5 list.
  * @param entry
  *   The new allocated entry, NULL if list entry size unspecified,
  *   New entry has to be allocated in callback and return.
@@ -354,46 +354,46 @@ typedef int (*mlx5_cache_match_cb)(struct mlx5_cache_list *list,
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_cache_entry *(*mlx5_cache_create_cb)
-				 (struct mlx5_cache_list *list,
-				  struct mlx5_cache_entry *entry,
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry,
 				  void *ctx);
 
 /**
- * Linked cache list structure.
+ * Linked mlx5 list structure.
  *
- * Entry in cache list could be reused if entry already exists,
+ * Entry in mlx5 list could be reused if entry already exists,
  * reference count will increase and the existing entry returns.
  *
  * When destroy an entry from list, decrease reference count and only
  * destroy when no further reference.
  *
- * Linked list cache is designed for limited number of entries cache,
+ * Linked list is designed for limited number of entries,
  * read mostly, less modification.
  *
- * For huge amount of entries cache, please consider hash list cache.
+ * For huge amount of entries, please consider hash list.
  *
  */
-struct mlx5_cache_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the cache list. */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	mlx5_cache_create_cb cb_create; /**< entry create callback. */
-	mlx5_cache_match_cb cb_match; /**< entry match callback. */
-	mlx5_cache_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_cache_head, mlx5_cache_entry) head;
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
 };
 
 /**
- * Initialize a cache list.
+ * Create a mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
  * @param name
- *   Name of the cache list.
+ *   Name of the mlx5 list.
  * @param entry_size
  *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
@@ -407,11 +407,11 @@ struct mlx5_cache_list {
  * @return
  *   0 on success, otherwise failure.
  */
-int mlx5_cache_list_init(struct mlx5_cache_list *list,
+int mlx5_list_create(struct mlx5_list *list,
 			 const char *name, uint32_t entry_size, void *ctx,
-			 mlx5_cache_create_cb cb_create,
-			 mlx5_cache_match_cb cb_match,
-			 mlx5_cache_remove_cb cb_remove);
+			 mlx5_list_create_cb cb_create,
+			 mlx5_list_match_cb cb_match,
+			 mlx5_list_remove_cb cb_remove);
 
 /**
  * Search an entry matching the key.
@@ -420,18 +420,18 @@ int mlx5_cache_list_init(struct mlx5_cache_list *list,
  * this function only in main thread.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  * @param ctx
  *   Common context parameter used by entry callback function.
  *
  * @return
- *   Pointer of the cache entry if found, NULL otherwise.
+ *   Pointer of the list entry if found, NULL otherwise.
  */
-struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
 					   void *ctx);
 
 /**
- * Reuse or create an entry to the cache list.
+ * Reuse or create an entry to the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list table.
@@ -441,42 +441,42 @@ struct mlx5_cache_entry *mlx5_cache_lookup(struct mlx5_cache_list *list,
  * @return
  *   registered entry on success, NULL otherwise
  */
-struct mlx5_cache_entry *mlx5_cache_register(struct mlx5_cache_list *list,
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
 					     void *ctx);
 
 /**
- * Remove an entry from the cache list.
+ * Remove an entry from the mlx5 list.
  *
  * User should guarantee the validity of the entry.
  *
  * @param list
  *   Pointer to the hast list.
  * @param entry
- *   Entry to be removed from the cache list table.
+ *   Entry to be removed from the mlx5 list table.
  * @return
  *   0 on entry removed, 1 on entry still referenced.
  */
-int mlx5_cache_unregister(struct mlx5_cache_list *list,
-			  struct mlx5_cache_entry *entry);
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
 
 /**
- * Destroy the cache list.
+ * Destroy the mlx5 list.
  *
  * @param list
- *   Pointer to the cache list.
+ *   Pointer to the mlx5 list.
  */
-void mlx5_cache_list_destroy(struct mlx5_cache_list *list);
+void mlx5_list_destroy(struct mlx5_list *list);
 
 /**
- * Get entry number from the cache list.
+ * Get entry number from the mlx5 list.
  *
  * @param list
  *   Pointer to the hast list.
  * @return
- *   Cache list entry number.
+ *   mlx5 list entry number.
  */
 uint32_t
-mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list);
+mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /********************************* indexed pool *************************/
 
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 17716b66c9..bcf72dc6db 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,10 +610,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v6 08/26] net/mlx5: add per lcore cache to the list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When mlx5 list object is accessed by multiple cores, the list lock
counter is all the time written by all the cores what increases cache
misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup
operation, all the other threads coming to do an operation in the list
are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when
the list objects are mostly reused.

Synchronization with atomic operations should be done in order to
allow threads to unregister an entry from other thread cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 5d537fa3e4..25e4bdb7a6 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1713,10 +1721,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index de5cd2e06e..1141e12977 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1388,6 +1388,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index cbae4f1872..2016adc6ee 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1646,7 +1646,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1654,6 +1658,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1662,6 +1671,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1670,6 +1684,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 3aa3192dd7..2d0e82fab7 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3826,6 +3826,39 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_port_id_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PORT_ID], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate port_id action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_port_id_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
+}
+
 /**
  * Find existing table port ID resource or create and register a new one.
  *
@@ -3918,6 +3951,39 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry __rte_unused,
+			  void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource;
+	uint32_t idx;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_PUSH_VLAN], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate push_vlan action memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	return &resource->entry;
+}
+
+void
+flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+			    struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_dv_push_vlan_action_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
+}
+
 /**
  * Find existing push vlan resource or create and register a new one.
  *
@@ -9892,6 +9958,36 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 	return match_criteria_enable;
 }
 
+static struct mlx5_list_entry *
+flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_matcher *ref = ctx->data;
+	struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
+							    typeof(*tbl), tbl);
+	struct mlx5_flow_dv_matcher *resource = mlx5_malloc(MLX5_MEM_ANY,
+							    sizeof(*resource),
+							    0, SOCKET_ID_ANY);
+
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot create matcher");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->tbl = &tbl->tbl;
+	return &resource->entry;
+}
+
+static void
+flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 struct mlx5_hlist_entry *
 flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 {
@@ -9958,10 +10054,12 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, 0, sh,
+	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
 			 flow_dv_matcher_create_cb,
 			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb);
+			 flow_dv_matcher_remove_cb,
+			 flow_dv_matcher_clone_cb,
+			 flow_dv_matcher_clone_free_cb);
 	return &tbl_data->entry;
 }
 
@@ -10749,6 +10847,45 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 
 }
 
+struct mlx5_list_entry *
+flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_sample_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t idx = 0;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_SAMPLE], &idx);
+	if (!resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_sample_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
+			resource->idx);
+}
+
 /**
  * Find existing sample resource or create and register a new one.
  *
@@ -10924,6 +11061,46 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 	return NULL;
 }
 
+struct mlx5_list_entry *
+flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct rte_eth_dev *dev = ctx->dev;
+	struct mlx5_flow_dv_dest_array_resource *resource;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_dev_ctx_shared *sh = priv->sh;
+	uint32_t res_idx = 0;
+	struct rte_flow_error *error = ctx->error;
+
+	resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DEST_ARRAY],
+				      &res_idx);
+	if (!resource) {
+		rte_flow_error_set(error, ENOMEM,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "cannot allocate dest-array memory");
+		return NULL;
+	}
+	memcpy(resource, entry, sizeof(*resource));
+	resource->idx = res_idx;
+	resource->dev = dev;
+	return &resource->entry;
+}
+
+void
+flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_dv_dest_array_resource *resource =
+			container_of(entry, typeof(*resource), entry);
+	struct rte_eth_dev *dev = resource->dev;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY], resource->idx);
+}
+
 /**
  * Find existing destination array resource or create and register a new one.
  *
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 3dcc71d51d..5450ddd388 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 8395332507..f8769da8dc 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1857,20 +1857,18 @@ mlx5_ind_table_obj_get(struct rte_eth_dev *dev, const uint16_t *queues,
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_ind_table_obj *ind_tbl;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		if ((ind_tbl->queues_n == queues_n) &&
 		    (memcmp(ind_tbl->queues, queues,
 			    ind_tbl->queues_n * sizeof(ind_tbl->queues[0]))
-		     == 0))
+		     == 0)) {
+			__atomic_fetch_add(&ind_tbl->refcnt, 1,
+					   __ATOMIC_RELAXED);
 			break;
+		}
 	}
-	if (ind_tbl) {
-		unsigned int i;
-
-		__atomic_fetch_add(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
-		for (i = 0; i != ind_tbl->queues_n; ++i)
-			mlx5_rxq_get(dev, ind_tbl->queues[i]);
-	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ind_tbl;
 }
 
@@ -1893,19 +1891,20 @@ mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
 			   bool standalone)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	unsigned int i;
+	unsigned int i, ret;
 
-	if (__atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED) == 0)
-		priv->obj_ops.ind_table_destroy(ind_tbl);
+	rte_rwlock_write_lock(&priv->ind_tbls_lock);
+	ret = __atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED);
+	if (!ret && !standalone)
+		LIST_REMOVE(ind_tbl, next);
+	rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	if (ret)
+		return 1;
+	priv->obj_ops.ind_table_destroy(ind_tbl);
 	for (i = 0; i != ind_tbl->queues_n; ++i)
 		claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
-	if (__atomic_load_n(&ind_tbl->refcnt, __ATOMIC_RELAXED) == 0) {
-		if (!standalone)
-			LIST_REMOVE(ind_tbl, next);
-		mlx5_free(ind_tbl);
-		return 0;
-	}
-	return 1;
+	mlx5_free(ind_tbl);
+	return 0;
 }
 
 /**
@@ -1924,12 +1923,14 @@ mlx5_ind_table_obj_verify(struct rte_eth_dev *dev)
 	struct mlx5_ind_table_obj *ind_tbl;
 	int ret = 0;
 
+	rte_rwlock_read_lock(&priv->ind_tbls_lock);
 	LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
 		DRV_LOG(DEBUG,
 			"port %u indirection table obj %p still referenced",
 			dev->data->port_id, (void *)ind_tbl);
 		++ret;
 	}
+	rte_rwlock_read_unlock(&priv->ind_tbls_lock);
 	return ret;
 }
 
@@ -2015,8 +2016,11 @@ mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues,
 		mlx5_free(ind_tbl);
 		return NULL;
 	}
-	if (!standalone)
+	if (!standalone) {
+		rte_rwlock_write_lock(&priv->ind_tbls_lock);
 		LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
+		rte_rwlock_write_unlock(&priv->ind_tbls_lock);
+	}
 	return ind_tbl;
 }
 
@@ -2328,6 +2332,35 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 	return hrxq ? &hrxq->entry : NULL;
 }
 
+struct mlx5_list_entry *
+mlx5_hrxq_clone_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry,
+		    void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq;
+	uint32_t hrxq_idx = 0;
+
+	hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx);
+	if (!hrxq)
+		return NULL;
+	memcpy(hrxq, entry, sizeof(*hrxq) + MLX5_RSS_HASH_KEY_LEN);
+	hrxq->idx = hrxq_idx;
+	return &hrxq->entry;
+}
+
+void
+mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
+{
+	struct rte_eth_dev *dev = list->ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq->idx);
+}
+
 /**
  * Get an Rx Hash queue.
  *
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 4536ca807d..f505caed4e 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -9,57 +9,68 @@
 #include "mlx5_utils.h"
 
 
-/********************* MLX5 list ************************/
-
-static struct mlx5_list_entry *
-mlx5_list_default_create_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_list_default_remove_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
-{
-	mlx5_free(entry);
-}
+/********************* mlx5 list ************************/
 
 int
-mlx5_list_create(struct mlx5_list *list, const char *name,
-		     uint32_t entry_size, void *ctx,
-		     mlx5_list_create_cb cb_create,
-		     mlx5_list_match_cb cb_match,
-		     mlx5_list_remove_cb cb_remove)
+mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	int i;
+
 	MLX5_ASSERT(list);
-	if (!cb_match || (!cb_create ^ !cb_remove))
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free)
 		return -1;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->entry_sz = entry_size;
 	list->ctx = ctx;
-	list->cb_create = cb_create ? cb_create : mlx5_list_default_create_cb;
+	list->cb_create = cb_create;
 	list->cb_match = cb_match;
-	list->cb_remove = cb_remove ? cb_remove : mlx5_list_default_remove_cb;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	LIST_INIT(&list->head);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry;
-
-	LIST_FOREACH(entry, &list->head, next) {
-		if (list->cb_match(list, entry, ctx))
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (list->cb_match(list, entry, ctx)) {
+			if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_ACQUIRE);
+				if (ret == 0) {
+					LIST_REMOVE(entry, next);
+					list->cb_clone_free(list, entry);
+				}
+			}
+			entry = nentry;
 			continue;
+		}
 		if (reuse) {
-			__atomic_add_fetch(&entry->ref_cnt, 1,
-					   __ATOMIC_RELAXED);
+			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+						 __ATOMIC_ACQUIRE);
+			if (ret == 1u) {
+				/* Entry was invalid before, free it. */
+				LIST_REMOVE(entry, next);
+				list->cb_clone_free(list, entry);
+				entry = nentry;
+				continue;
+			}
 			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
 				list->name, (void *)entry, entry->ref_cnt);
 		}
@@ -68,96 +79,141 @@ __list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
 	return entry;
 }
 
-static struct mlx5_list_entry *
-list_lookup(struct mlx5_list *list, void *ctx, bool reuse)
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry = NULL;
+	int i;
 
 	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, ctx, reuse);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
 	rte_rwlock_read_unlock(&list->lock);
 	return entry;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	return list_lookup(list, ctx, false);
+	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+
+	if (!lentry)
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
 }
 
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry;
+	struct mlx5_list_entry *entry, *lentry;
 	uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
-	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* Lookup in local cache. */
+	lentry = __list_lookup(list, lcore_index, ctx, true);
+	if (lentry)
+		return lentry;
 	/* Lookup with read lock, reuse if found. */
-	entry = list_lookup(list, ctx, true);
-	if (entry)
-		return entry;
+	rte_rwlock_read_lock(&list->lock);
+	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+	if (entry == NULL) {
+		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
+					       __ATOMIC_ACQUIRE);
+		rte_rwlock_read_unlock(&list->lock);
+	} else {
+		rte_rwlock_read_unlock(&list->lock);
+		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+	}
 	/* Not found, append with write lock - block read from other threads. */
 	rte_rwlock_write_lock(&list->lock);
 	/* If list changed by other threads before lock, search again. */
 	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
 		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, ctx, true);
-		if (entry)
-			goto done;
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (entry) {
+			rte_rwlock_write_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
 	}
 	entry = list->cb_create(list, entry, ctx);
-	if (!entry) {
-		DRV_LOG(ERR, "Failed to init mlx5 list %s entry %p.",
-			list->name, (void *)entry);
-		goto done;
+	if (entry) {
+		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+		if (!lentry) {
+			list->cb_remove(list, entry);
+		} else {
+			entry->ref_cnt = 1u;
+			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
+					 next);
+			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
+			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+				list->name, (void *)entry, entry->ref_cnt);
+		}
+
 	}
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(&list->head, entry, next);
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-done:
 	rte_rwlock_write_unlock(&list->lock);
-	return entry;
+	return lentry;
 }
 
 int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
+	struct mlx5_list_entry *gentry = entry->gentry;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	MLX5_ASSERT(entry && entry->next.le_prev);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p ref--: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
+	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
+		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		LIST_REMOVE(gentry, next);
+		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
-		return 1;
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
 	}
-	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-	__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	LIST_REMOVE(entry, next);
-	list->cb_remove(list, entry);
 	rte_rwlock_write_unlock(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-		list->name, (void *)entry);
-	return 0;
+	return 1;
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
+	int i;
 
 	MLX5_ASSERT(list);
-	/* no LIST_FOREACH_SAFE, using while instead */
-	while (!LIST_EMPTY(&list->head)) {
-		entry = LIST_FIRST(&list->head);
-		LIST_REMOVE(entry, next);
-		list->cb_remove(list, entry);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p destroyed.",
-			list->name, (void *)entry);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list, entry);
+			}
+		}
 	}
 	memset(list, 0, sizeof(*list));
 }
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index cfb3cb6180..9e3fe0cb85 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -310,9 +310,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -340,6 +345,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -376,15 +388,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -394,8 +408,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -408,10 +420,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index bcf72dc6db..8ced98f0dc 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,9 +610,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
-- 
2.25.1

[dpdk-dev] [PATCH v6 09/26] net/mlx5: minimize list critical sections

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is
taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical
sections.

These operations are heavy and make the critical sections heavy because
they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation
counter in order to detect parallel allocations.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index f505caed4e..c4c9adb039 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 9e3fe0cb85..6dade8238d 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -388,8 +388,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
-- 
2.25.1

[dpdk-dev] [PATCH v6 10/26] net/mlx5: manage list cache entries release

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

When a cache entry is allocated by lcore A and is released by lcore B,
the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be
increased atomically when the non-original lcore decreases the reference
counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in
its cache, it will check the counter in order to free invalid entries in
its cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index c4c9adb039..13c7dbe1c2 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 6dade8238d..71da5ab4f9 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -311,11 +311,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v6 11/26] net/mlx5: relax the list utility atomic operations

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

The atomic operation in the list utility no need a barriers because the
critical part are managed by RW lock.

Relax them.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 13c7dbe1c2..daecf37575 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
-- 
2.25.1

[dpdk-dev] [PATCH v6 12/26] net/mlx5: allocate list memory by the create API

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence
with the hlist utility.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  60 ++++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 139 insertions(+), 83 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 25e4bdb7a6..2a9a6c3bf8 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -456,6 +464,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -517,9 +541,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1721,11 +1759,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1784,7 +1824,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 5cd2cfb07c..d04b5d616f 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1612,7 +1612,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 1141e12977..88b84d749a 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1137,10 +1137,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1382,7 +1382,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2016adc6ee..59a6f31d2e 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -591,7 +591,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 2d0e82fab7..5a536e3dff 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3889,7 +3889,7 @@ flow_dv_port_id_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->port_id_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->port_id_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -4014,7 +4014,7 @@ flow_dv_push_vlan_action_resource_register
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->push_vlan_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->push_vlan_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10054,12 +10054,22 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	mlx5_list_create(&tbl_data->matchers, matcher_name, sh,
-			 flow_dv_matcher_create_cb,
-			 flow_dv_matcher_match_cb,
-			 flow_dv_matcher_remove_cb,
-			 flow_dv_matcher_clone_cb,
-			 flow_dv_matcher_clone_free_cb);
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+					      flow_dv_matcher_create_cb,
+					      flow_dv_matcher_match_cb,
+					      flow_dv_matcher_remove_cb,
+					      flow_dv_matcher_clone_cb,
+					      flow_dv_matcher_clone_free_cb);
+	if (!tbl_data->matchers) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot create tbl matcher list");
+		mlx5_flow_os_destroy_flow_action(tbl_data->jump.action);
+		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
+		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
+		return NULL;
+	}
 	return &tbl_data->entry;
 }
 
@@ -10187,7 +10197,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_list_destroy(&tbl_data->matchers);
+	mlx5_list_destroy(tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10319,7 +10329,7 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
 	ref->tbl = tbl;
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		flow_dv_tbl_resource_release(MLX5_SH(dev), tbl);
 		return rte_flow_error_set(error, ENOMEM,
@@ -10916,7 +10926,7 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->sample_action_list, &ctx);
+	entry = mlx5_list_register(priv->sh->sample_action_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -11131,7 +11141,7 @@ flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
 		.data = ref,
 	};
 
-	entry = mlx5_list_register(&priv->sh->dest_array_list, &ctx);
+	entry = mlx5_list_register(priv->sh->dest_array_list, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -13610,7 +13620,7 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	int ret;
 
 	MLX5_ASSERT(matcher->matcher_object);
-	ret = mlx5_list_unregister(&tbl->matchers, &matcher->entry);
+	ret = mlx5_list_unregister(tbl->matchers, &matcher->entry);
 	flow_dv_tbl_resource_release(MLX5_SH(dev), &tbl->tbl);
 	return ret;
 }
@@ -13753,7 +13763,7 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->port_id_action_list,
+	return mlx5_list_unregister(priv->sh->port_id_action_list,
 				    &resource->entry);
 }
 
@@ -13811,7 +13821,7 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->push_vlan_action_list,
+	return mlx5_list_unregister(priv->sh->push_vlan_action_list,
 				    &resource->entry);
 }
 
@@ -13892,7 +13902,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->verbs_action);
-	return mlx5_list_unregister(&priv->sh->sample_action_list,
+	return mlx5_list_unregister(priv->sh->sample_action_list,
 				    &resource->entry);
 }
 
@@ -13940,7 +13950,7 @@ flow_dv_dest_array_resource_release(struct rte_eth_dev *dev,
 	if (!resource)
 		return 0;
 	MLX5_ASSERT(resource->action);
-	return mlx5_list_unregister(&priv->sh->dest_array_list,
+	return mlx5_list_unregister(priv->sh->dest_array_list,
 				    &resource->entry);
 }
 
@@ -14791,7 +14801,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 			claim_zero(mlx5_flow_os_destroy_flow(color_rule->rule));
 			tbl = container_of(color_rule->matcher->tbl,
 					typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 						&color_rule->matcher->entry);
 			TAILQ_REMOVE(&sub_policy->color_rules[i],
 					color_rule, next_port);
@@ -15584,7 +15594,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 		if (mtrmng->def_matcher[i]) {
 			tbl = container_of(mtrmng->def_matcher[i]->tbl,
 				struct mlx5_flow_tbl_data_entry, tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 					     &mtrmng->def_matcher[i]->entry);
 			mtrmng->def_matcher[i] = NULL;
 		}
@@ -15594,7 +15604,7 @@ flow_dv_destroy_mtr_drop_tbls(struct rte_eth_dev *dev)
 				container_of(mtrmng->drop_matcher[i][j]->tbl,
 					     struct mlx5_flow_tbl_data_entry,
 					     tbl);
-				mlx5_list_unregister(&tbl->matchers,
+				mlx5_list_unregister(tbl->matchers,
 					    &mtrmng->drop_matcher[i][j]->entry);
 				mtrmng->drop_matcher[i][j] = NULL;
 			}
@@ -15729,7 +15739,7 @@ __flow_dv_create_policy_matcher(struct rte_eth_dev *dev,
 	matcher.priority = priority;
 	matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
 					matcher.mask.size);
-	entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+	entry = mlx5_list_register(tbl_data->matchers, &ctx);
 	if (!entry) {
 		DRV_LOG(ERR, "Failed to register meter drop matcher.");
 		return -1;
@@ -15837,7 +15847,7 @@ __flow_dv_create_domain_policy_rules(struct rte_eth_dev *dev,
 			struct mlx5_flow_tbl_data_entry *tbl =
 				container_of(color_rule->matcher->tbl,
 						typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 						&color_rule->matcher->entry);
 		}
 		mlx5_free(color_rule);
@@ -16235,7 +16245,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR, "Failed to register meter "
 				"drop default matcher.");
@@ -16272,7 +16282,7 @@ flow_dv_create_mtr_tbls(struct rte_eth_dev *dev,
 			matcher.crc = rte_raw_cksum
 					((const void *)matcher.mask.buf,
 					matcher.mask.size);
-			entry = mlx5_list_register(&tbl_data->matchers, &ctx);
+			entry = mlx5_list_register(tbl_data->matchers, &ctx);
 			if (!entry) {
 				DRV_LOG(ERR,
 				"Failed to register meter drop matcher.");
@@ -16689,7 +16699,7 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
 			struct mlx5_flow_tbl_data_entry *tbl =
 				container_of(color_rule->matcher->tbl,
 						typeof(*tbl), tbl);
-			mlx5_list_unregister(&tbl->matchers,
+			mlx5_list_unregister(tbl->matchers,
 						&color_rule->matcher->entry);
 		}
 		mlx5_free(color_rule);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index f8769da8dc..aa9e973d10 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index daecf37575..0be778935f 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 71da5ab4f9..ea64bb75c9 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -420,15 +420,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 8ced98f0dc..e6176e70d2 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v6 13/26] common/mlx5: move list utility to common

[Suanming Mou]

Hash list is planned to be implemented with the cache list code.

This commit moves the list utility to common directory.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 250 +++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h | 205 +++++++++++++++++++
 drivers/common/mlx5/version.map         |   7 +
 drivers/net/mlx5/mlx5_utils.c           | 251 ------------------------
 drivers/net/mlx5/mlx5_utils.h           | 197 -------------------
 6 files changed, 464 insertions(+), 448 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 306f2f1ab7..7fb7d40b38 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index ad2011e858..8bb8a6016d 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -11,6 +11,256 @@
 #include "mlx5_common_utils.h"
 #include "mlx5_common_log.h"
 
+/********************* mlx5 list ************************/
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+	int i;
+
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (name)
+		snprintf(list->name, sizeof(list->name), "%s", name);
+	list->ctx = ctx;
+	list->cb_create = cb_create;
+	list->cb_match = cb_match;
+	list->cb_remove = cb_remove;
+	list->cb_clone = cb_clone;
+	list->cb_clone_free = cb_clone_free;
+	rte_rwlock_init(&list->lock);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	for (i = 0; i <= RTE_MAX_LCORE; i++)
+		LIST_INIT(&list->cache[i].h);
+	return list;
+}
+
+static struct mlx5_list_entry *
+__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+{
+	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	uint32_t ret;
+
+	while (entry != NULL) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_RELAXED) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
+				ret = __atomic_load_n(&entry->ref_cnt,
+						      __ATOMIC_RELAXED);
+			}
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
+		}
+		entry = LIST_NEXT(entry, next);
+	}
+	return NULL;
+}
+
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry = NULL;
+	int i;
+
+	rte_rwlock_read_lock(&list->lock);
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		entry = __list_lookup(list, i, ctx, false);
+		if (entry)
+			break;
+	}
+	rte_rwlock_read_unlock(&list->lock);
+	return entry;
+}
+
+static struct mlx5_list_entry *
+mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+		       struct mlx5_list_entry *gentry, void *ctx)
+{
+	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+
+	if (unlikely(!lentry))
+		return NULL;
+	lentry->ref_cnt = 1u;
+	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	return lentry;
+}
+
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	MLX5_ASSERT(list);
+	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
+	if (unlikely(lcore_index == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
+	rte_rwlock_read_lock(&list->lock);
+	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+	if (likely(entry)) {
+		rte_rwlock_read_unlock(&list->lock);
+		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
+	}
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
+	rte_rwlock_write_lock(&list->lock);
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
+			rte_rwlock_write_unlock(&list->lock);
+			list->cb_remove(list, entry);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
+		}
+	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
+	rte_rwlock_write_unlock(&list->lock);
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
+	return local_entry;
+}
+
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
+
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
+		return 1;
+	rte_rwlock_write_lock(&list->lock);
+	if (likely(gentry->ref_cnt == 0)) {
+		LIST_REMOVE(gentry, next);
+		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)gentry);
+		return 0;
+	}
+	rte_rwlock_write_unlock(&list->lock);
+	return 1;
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	struct mlx5_list_entry *entry;
+	int i;
+
+	MLX5_ASSERT(list);
+	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+		while (!LIST_EMPTY(&list->cache[i].h)) {
+			entry = LIST_FIRST(&list->cache[i].h);
+			LIST_REMOVE(entry, next);
+			if (i == RTE_MAX_LCORE) {
+				list->cb_remove(list, entry);
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
+					"destroyed.", list->name,
+					(void *)entry);
+			} else {
+				list->cb_clone_free(list, entry);
+			}
+		}
+	}
+	mlx5_free(list);
+}
+
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list)
+{
+	MLX5_ASSERT(list);
+	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+}
+
 /********************* Hash List **********************/
 
 static struct mlx5_hlist_entry *
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index ed378ce9bd..96add6d003 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -7,6 +7,211 @@
 
 #include "mlx5_common.h"
 
+/************************ mlx5 list *****************************/
+
+/** Maximum size of string for naming. */
+#define MLX5_NAME_SIZE			32
+
+struct mlx5_list;
+
+/**
+ * Structure of the entry in the mlx5 list, user should define its own struct
+ * that contains this in order to store the data.
+ */
+struct mlx5_list_entry {
+	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
+	struct mlx5_list_entry *gentry;
+};
+
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
+} __rte_cache_aligned;
+
+/**
+ * Type of callback function for entry removal.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The entry in the list.
+ */
+typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
+				     struct mlx5_list_entry *entry);
+
+/**
+ * Type of function for user defined matching.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The entry in the list.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   0 if matching, non-zero number otherwise.
+ */
+typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+				   struct mlx5_list_entry *entry, void *ctx);
+
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
+/**
+ * Type of function for user defined mlx5 list entry creation.
+ *
+ * @param list
+ *   The mlx5 list.
+ * @param entry
+ *   The new allocated entry, NULL if list entry size unspecified,
+ *   New entry has to be allocated in callback and return.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   Pointer of entry on success, NULL otherwise.
+ */
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry,
+				  void *ctx);
+
+/**
+ * Linked mlx5 list structure.
+ *
+ * Entry in mlx5 list could be reused if entry already exists,
+ * reference count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ *
+ * Linked list is designed for limited number of entries,
+ * read mostly, less modification.
+ *
+ * For huge amount of entries, please consider hash list.
+ *
+ */
+struct mlx5_list {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
+	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+};
+
+/**
+ * Create a mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param name
+ *   Name of the mlx5 list.
+ * @param ctx
+ *   Pointer to the list context data.
+ * @param cb_create
+ *   Callback function for entry create.
+ * @param cb_match
+ *   Callback function for entry match.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @return
+ *   List pointer on success, otherwise NULL.
+ */
+__rte_internal
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ * @param ctx
+ *   Common context parameter used by entry callback function.
+ *
+ * @return
+ *   Pointer of the list entry if found, NULL otherwise.
+ */
+__rte_internal
+struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
+					   void *ctx);
+
+/**
+ * Reuse or create an entry to the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list table.
+ * @param ctx
+ *   Common context parameter used by callback function.
+ *
+ * @return
+ *   registered entry on success, NULL otherwise
+ */
+__rte_internal
+struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
+					     void *ctx);
+
+/**
+ * Remove an entry from the mlx5 list.
+ *
+ * User should guarantee the validity of the entry.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @param entry
+ *   Entry to be removed from the mlx5 list table.
+ * @return
+ *   0 on entry removed, 1 on entry still referenced.
+ */
+__rte_internal
+int mlx5_list_unregister(struct mlx5_list *list,
+			  struct mlx5_list_entry *entry);
+
+/**
+ * Destroy the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the mlx5 list.
+ */
+__rte_internal
+void mlx5_list_destroy(struct mlx5_list *list);
+
+/**
+ * Get entry number from the mlx5 list.
+ *
+ * @param list
+ *   Pointer to the hast list.
+ * @return
+ *   mlx5 list entry number.
+ */
+__rte_internal
+uint32_t
+mlx5_list_get_entry_num(struct mlx5_list *list);
+
+/************************ Hash list *****************************/
+
 #define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
 #define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
 
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index b8be73a77b..e6586d6f6f 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -73,6 +73,13 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_list_create;
+	mlx5_list_register;
+	mlx5_list_unregister;
+	mlx5_list_lookup;
+	mlx5_list_get_entry_num;
+	mlx5_list_destroy;
+
 	mlx5_hlist_create;
 	mlx5_hlist_lookup;
 	mlx5_hlist_register;
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 0be778935f..e4e66ae4c5 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -8,257 +8,6 @@
 
 #include "mlx5_utils.h"
 
-
-/********************* mlx5 list ************************/
-
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
-{
-	struct mlx5_list *list;
-	int i;
-
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return NULL;
-	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
-	return list;
-}
-
-static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
-{
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
-	uint32_t ret;
-
-	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
-			if (reuse) {
-				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_RELAXED) - 1;
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
-					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
-				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_RELAXED);
-			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
-				return entry;
-			if (reuse && ret == 0)
-				entry->ref_cnt--; /* Invalid entry. */
-		}
-		entry = LIST_NEXT(entry, next);
-	}
-	return NULL;
-}
-
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry = NULL;
-	int i;
-
-	rte_rwlock_read_lock(&list->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
-		if (entry)
-			break;
-	}
-	rte_rwlock_read_unlock(&list->lock);
-	return entry;
-}
-
-static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
-		       struct mlx5_list_entry *gentry, void *ctx)
-{
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
-
-	if (unlikely(!lentry))
-		return NULL;
-	lentry->ref_cnt = 1u;
-	lentry->gentry = gentry;
-	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
-	return lentry;
-}
-
-static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
-{
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
-	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
-	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
-					       __ATOMIC_RELAXED);
-
-	while (inv_cnt != 0 && entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
-			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
-			inv_cnt--;
-		}
-		entry = nentry;
-	}
-}
-
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
-{
-	struct mlx5_list_entry *entry, *local_entry;
-	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
-	MLX5_ASSERT(list);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
-	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
-	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
-	if (local_entry)
-		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
-		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
-	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
-	if (unlikely(!entry))
-		return NULL;
-	local_entry = list->cb_clone(list, entry, ctx);
-	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
-		return NULL;
-	}
-	entry->ref_cnt = 1u;
-	local_entry->ref_cnt = 1u;
-	local_entry->gentry = entry;
-	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
-	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
-							       RTE_MAX_LCORE,
-							       ctx, true);
-
-		if (unlikely(oentry)) {
-			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
-		}
-	}
-	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
-		(void *)entry, entry->ref_cnt);
-	return local_entry;
-}
-
-int
-mlx5_list_unregister(struct mlx5_list *list,
-		      struct mlx5_list_entry *entry)
-{
-	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
-
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
-	if (entry->lcore_idx == (uint32_t)lcore_idx) {
-		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
-	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
-				   __ATOMIC_RELAXED);
-	} else {
-		return 0;
-	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
-		return 1;
-	rte_rwlock_write_lock(&list->lock);
-	if (likely(gentry->ref_cnt == 0)) {
-		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
-		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
-		return 0;
-	}
-	rte_rwlock_write_unlock(&list->lock);
-	return 1;
-}
-
-void
-mlx5_list_destroy(struct mlx5_list *list)
-{
-	struct mlx5_list_entry *entry;
-	int i;
-
-	MLX5_ASSERT(list);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
-			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
-				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
-					(void *)entry);
-			} else {
-				list->cb_clone_free(list, entry);
-			}
-		}
-	}
-	mlx5_free(list);
-}
-
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list)
-{
-	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
-}
-
 /********************* Indexed pool **********************/
 
 static inline void
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index ea64bb75c9..cf3db89403 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -297,203 +297,6 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-/************************ mlx5 list *****************************/
-
-/** Maximum size of string for naming. */
-#define MLX5_NAME_SIZE			32
-
-struct mlx5_list;
-
-/**
- * Structure of the entry in the mlx5 list, user should define its own struct
- * that contains this in order to store the data.
- */
-struct mlx5_list_entry {
-	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* 0 means, entry is invalid. */
-	uint32_t lcore_idx;
-	struct mlx5_list_entry *gentry;
-};
-
-struct mlx5_list_cache {
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
-	uint32_t inv_cnt; /* Invalid entries counter. */
-} __rte_cache_aligned;
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The entry in the list.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
-				   struct mlx5_list_entry *entry, void *ctx);
-
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
-
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
-
-/**
- * Type of function for user defined mlx5 list entry creation.
- *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer of entry on success, NULL otherwise.
- */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
-
-/**
- * Linked mlx5 list structure.
- *
- * Entry in mlx5 list could be reused if entry already exists,
- * reference count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- *
- * Linked list is designed for limited number of entries,
- * read mostly, less modification.
- *
- * For huge amount of entries, please consider hash list.
- *
- */
-struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
-	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-};
-
-/**
- * Create a mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param name
- *   Name of the mlx5 list.
- * @param ctx
- *   Pointer to the list context data.
- * @param cb_create
- *   Callback function for entry create.
- * @param cb_match
- *   Callback function for entry match.
- * @param cb_remove
- *   Callback function for entry remove.
- * @return
- *   List pointer on success, otherwise NULL.
- */
-struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
-				   mlx5_list_create_cb cb_create,
-				   mlx5_list_match_cb cb_match,
-				   mlx5_list_remove_cb cb_remove,
-				   mlx5_list_clone_cb cb_clone,
-				   mlx5_list_clone_free_cb cb_clone_free);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param list
- *   Pointer to the mlx5 list.
- * @param ctx
- *   Common context parameter used by entry callback function.
- *
- * @return
- *   Pointer of the list entry if found, NULL otherwise.
- */
-struct mlx5_list_entry *mlx5_list_lookup(struct mlx5_list *list,
-					   void *ctx);
-
-/**
- * Reuse or create an entry to the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list table.
- * @param ctx
- *   Common context parameter used by callback function.
- *
- * @return
- *   registered entry on success, NULL otherwise
- */
-struct mlx5_list_entry *mlx5_list_register(struct mlx5_list *list,
-					     void *ctx);
-
-/**
- * Remove an entry from the mlx5 list.
- *
- * User should guarantee the validity of the entry.
- *
- * @param list
- *   Pointer to the hast list.
- * @param entry
- *   Entry to be removed from the mlx5 list table.
- * @return
- *   0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_list_unregister(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry);
-
-/**
- * Destroy the mlx5 list.
- *
- * @param list
- *   Pointer to the mlx5 list.
- */
-void mlx5_list_destroy(struct mlx5_list *list);
-
-/**
- * Get entry number from the mlx5 list.
- *
- * @param list
- *   Pointer to the hast list.
- * @return
- *   mlx5 list entry number.
- */
-uint32_t
-mlx5_list_get_entry_num(struct mlx5_list *list);
-
 /********************************* indexed pool *************************/
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v6 14/26] common/mlx5: add list lcore share

[Suanming Mou]

As some actions in SW-steering is only memory and can be allowed to
create duplicate objects, for lists which no need to check if there
are existing same objects in other sub local lists, search the object
only in local list will be more efficient.

This commit adds the lcore share mode to list optimized the list
register.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 48 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h | 16 ++++++---
 drivers/net/mlx5/linux/mlx5_os.c        | 11 +++---
 drivers/net/mlx5/mlx5_flow_dv.c         |  2 +-
 drivers/net/mlx5/windows/mlx5_os.c      |  2 +-
 5 files changed, 56 insertions(+), 23 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 8bb8a6016d..bc08f8ba25 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -14,7 +14,7 @@
 /********************* mlx5 list ************************/
 
 struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
@@ -35,6 +35,7 @@ mlx5_list_create(const char *name, void *ctx,
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
+	list->lcores_share = lcores_share;
 	list->cb_create = cb_create;
 	list->cb_match = cb_match;
 	list->cb_remove = cb_remove;
@@ -119,7 +120,10 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
+			if (list->lcores_share)
+				list->cb_clone_free(list, entry);
+			else
+				list->cb_remove(list, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -129,7 +133,7 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *local_entry;
+	struct mlx5_list_entry *entry = NULL, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
@@ -145,25 +149,36 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
 		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
 	/* 3. Prepare new entry for global list and for cache. */
 	entry = list->cb_create(list, entry, ctx);
 	if (unlikely(!entry))
 		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+		return entry;
+	}
 	local_entry = list->cb_clone(list, entry, ctx);
 	if (unlikely(!local_entry)) {
 		list->cb_remove(list, entry);
 		return NULL;
 	}
-	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
@@ -207,13 +222,22 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
+		if (list->lcores_share)
+			list->cb_clone_free(list, entry);
+		else
+			list->cb_remove(list, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 96add6d003..000279d236 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -100,11 +100,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -112,17 +109,27 @@ struct mlx5_list {
 	mlx5_list_clone_free_cb cb_clone_free;
 	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
 };
 
 /**
  * Create a mlx5 list.
  *
+ * For actions in SW-steering is only memory and  can be allowed
+ * to create duplicate objects, the lists don't need to check if
+ * there are existing same objects in other sub local lists,
+ * search the object only in local list will be more efficient.
+ *
  * @param list
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
  * @param ctx
  *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
@@ -134,6 +141,7 @@ struct mlx5_list {
  */
 __rte_internal
 struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
 				   mlx5_list_create_cb cb_create,
 				   mlx5_list_match_cb cb_match,
 				   mlx5_list_remove_cb cb_remove,
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 2a9a6c3bf8..ce41fb34a0 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 						   flow_dv_port_id_create_cb,
 						   flow_dv_port_id_match_cb,
 						   flow_dv_port_id_remove_cb,
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 						    flow_dv_push_vlan_create_cb,
 						    flow_dv_push_vlan_match_cb,
 						    flow_dv_push_vlan_remove_cb,
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 						  flow_dv_sample_remove_cb,
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -1759,7 +1759,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
 				       mlx5_hrxq_clone_cb,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 5a536e3dff..4a45172a12 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -10054,7 +10054,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
 					      flow_dv_matcher_remove_cb,
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index e6176e70d2..a04f93e1d4 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
-- 
2.25.1

[dpdk-dev] [PATCH v6 15/26] common/mlx5: call list callbacks with context

[Suanming Mou]

This commit optimizes to call the list callback functions with global
context directly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c |  24 ++---
 drivers/common/mlx5/mlx5_common_utils.h |  36 ++++----
 drivers/net/mlx5/mlx5_flow.h            |  94 +++++++++----------
 drivers/net/mlx5/mlx5_flow_dv.c         | 115 +++++++++++-------------
 drivers/net/mlx5/mlx5_rx.h              |  13 ++-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +++--------
 6 files changed, 132 insertions(+), 203 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index bc08f8ba25..0f400f6353 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -55,7 +55,7 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
+		if (list->cb_match(list->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
@@ -96,7 +96,7 @@ static struct mlx5_list_entry *
 mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
@@ -121,9 +121,9 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
 			if (list->lcores_share)
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			else
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -162,7 +162,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_rwlock_read_unlock(&list->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
+	entry = list->cb_create(list->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
@@ -174,9 +174,9 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list, entry, ctx);
+	local_entry = list->cb_clone(list->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
+		list->cb_remove(list->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
@@ -192,8 +192,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
+			list->cb_remove(list->ctx, entry);
+			list->cb_clone_free(list->ctx, local_entry);
 			return mlx5_list_cache_insert(list, lcore_index, oentry,
 						      ctx);
 		}
@@ -223,9 +223,9 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (list->lcores_share)
-			list->cb_clone_free(list, entry);
+			list->cb_clone_free(list->ctx, entry);
 		else
-			list->cb_remove(list, entry);
+			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
@@ -244,7 +244,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
+		list->cb_remove(list->ctx, gentry);
 		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 000279d236..a691ff8f0d 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -33,19 +33,19 @@ struct mlx5_list_cache {
 /**
  * Type of callback function for entry removal.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance user context.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_remove_cb)(void *tool_ctx,
+				    struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance context.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -54,34 +54,28 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+typedef int (*mlx5_list_match_cb)(void *tool_ctx,
 				   struct mlx5_list_entry *entry, void *ctx);
 
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)(void *tool_ctx,
+				      struct mlx5_list_entry *entry, void *ctx);
 
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
+					struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
+ * @param tool_ctx
+ *   The mlx5 tool instance context.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
+						       void *ctx);
 
 /**
  * Linked mlx5 list structure.
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 59a6f31d2e..70b3119d09 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1631,64 +1631,50 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
 
-int flow_dv_matcher_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *ctx);
-struct mlx5_list_entry *flow_dv_matcher_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *ctx);
-void flow_dv_matcher_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
-
-int flow_dv_port_id_match_cb(struct mlx5_list *list,
-			     struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused,
-				 void *cb_ctx);
-void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry __rte_unused);
-int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
-			       struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
-				(struct mlx5_list *list,
+int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *ctx);
+struct mlx5_list_entry *flow_dv_matcher_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_matcher_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_port_id_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			     void *cb_ctx);
+struct mlx5_list_entry *flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(void *tool_ctx,
+				struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry);
+
+int flow_dv_push_vlan_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			       void *cb_ctx);
+struct mlx5_list_entry *flow_dv_push_vlan_create_cb(void *tool_ctx,
+						    void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb(void *tool_ctx,
 				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry);
-
-int flow_dv_sample_match_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
-						 struct mlx5_list_entry *entry,
-						 void *cb_ctx);
-void flow_dv_sample_remove_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_sample_clone_cb
-				(struct mlx5_list *list,
+void flow_dv_push_vlan_clone_free_cb(void *tool_ctx,
+				     struct mlx5_list_entry *entry);
+
+int flow_dv_sample_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_sample_create_cb(void *tool_ctx, void *cb_ctx);
+void flow_dv_sample_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb(void *tool_ctx,
 				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
-			      struct mlx5_list_entry *entry);
-
-int flow_dv_dest_array_match_cb(struct mlx5_list *list,
-				struct mlx5_list_entry *entry, void *cb_ctx);
-struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
-						  struct mlx5_list_entry *entry,
-						  void *cb_ctx);
-void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
+void flow_dv_sample_clone_free_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_dest_array_clone_cb
-				(struct mlx5_list *list,
-				 struct mlx5_list_entry *entry, void *cb_ctx);
-void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+
+int flow_dv_dest_array_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+				void *cb_ctx);
+struct mlx5_list_entry *flow_dv_dest_array_create_cb(void *tool_ctx,
+						     void *cb_ctx);
+void flow_dv_dest_array_remove_cb(void *tool_ctx,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(void *tool_ctx,
+				      struct mlx5_list_entry *entry);
+
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 4a45172a12..795de65089 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3780,23 +3780,21 @@ flow_dv_jump_tbl_resource_register
 }
 
 int
-flow_dv_port_id_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_port_id_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->port_id != res->port_id;
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
@@ -3827,11 +3825,11 @@ flow_dv_port_id_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_port_id_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_port_id_clone_cb(void *tool_ctx,
+			 struct mlx5_list_entry *entry __rte_unused,
+			 void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource;
 	uint32_t idx;
@@ -3849,12 +3847,11 @@ flow_dv_port_id_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PORT_ID], resource->idx);
 }
@@ -3899,23 +3896,21 @@ flow_dv_port_id_action_resource_register
 }
 
 int
-flow_dv_push_vlan_match_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry, void *cb_ctx)
+flow_dv_push_vlan_match_cb(void *tool_ctx __rte_unused,
+			   struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *res =
-			container_of(entry, typeof(*res), entry);
+				       container_of(entry, typeof(*res), entry);
 
 	return ref->vlan_tag != res->vlan_tag || ref->ft_type != res->ft_type;
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *ref = ctx->data;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
@@ -3952,11 +3947,11 @@ flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_push_vlan_clone_cb(void *tool_ctx,
+			   struct mlx5_list_entry *entry __rte_unused,
+			   void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource;
 	uint32_t idx;
@@ -3974,12 +3969,11 @@ flow_dv_push_vlan_clone_cb(struct mlx5_list *list,
 }
 
 void
-flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_PUSH_VLAN], resource->idx);
 }
@@ -9959,7 +9953,7 @@ flow_dv_matcher_enable(uint32_t *match_criteria)
 }
 
 static struct mlx5_list_entry *
-flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -9982,7 +9976,7 @@ flow_dv_matcher_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 static void
-flow_dv_matcher_clone_free_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	mlx5_free(entry);
@@ -10225,7 +10219,7 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
 }
 
 int
-flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10240,11 +10234,9 @@ flow_dv_matcher_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_matcher_create_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry __rte_unused,
-			  void *cb_ctx)
+flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_matcher *ref = ctx->data;
 	struct mlx5_flow_dv_matcher *resource;
@@ -10740,7 +10732,7 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_match_cb(void *tool_ctx __rte_unused,
 			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -10769,9 +10761,7 @@ flow_dv_sample_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_sample_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -10858,7 +10848,7 @@ flow_dv_sample_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_clone_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry __rte_unused,
 			 void *cb_ctx)
 {
@@ -10884,16 +10874,15 @@ flow_dv_sample_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_sample_clone_free_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry)
+flow_dv_sample_clone_free_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	struct rte_eth_dev *dev = resource->dev;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
-			resource->idx);
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_SAMPLE], resource->idx);
 }
 
 /**
@@ -10936,14 +10925,14 @@ flow_dv_sample_resource_register(struct rte_eth_dev *dev,
 }
 
 int
-flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_match_cb(void *tool_ctx __rte_unused,
 			    struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_dest_array_resource *ctx_resource = ctx->data;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_dv_dest_array_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t idx = 0;
 
 	if (ctx_resource->num_of_dest == resource->num_of_dest &&
@@ -10965,9 +10954,7 @@ flow_dv_dest_array_match_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11072,9 +11059,9 @@ flow_dv_dest_array_create_cb(struct mlx5_list *list __rte_unused,
 }
 
 struct mlx5_list_entry *
-flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
-			 struct mlx5_list_entry *entry __rte_unused,
-			 void *cb_ctx)
+flow_dv_dest_array_clone_cb(void *tool_ctx __rte_unused,
+			    struct mlx5_list_entry *entry __rte_unused,
+			    void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
@@ -11100,8 +11087,8 @@ flow_dv_dest_array_clone_cb(struct mlx5_list *list __rte_unused,
 }
 
 void
-flow_dv_dest_array_clone_free_cb(struct mlx5_list *list __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_dest_array_clone_free_cb(void *tool_ctx __rte_unused,
+				 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
 			container_of(entry, typeof(*resource), entry);
@@ -13588,7 +13575,7 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
 }
 
 void
-flow_dv_matcher_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
 			  struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_matcher *resource = container_of(entry,
@@ -13730,10 +13717,9 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_port_id_remove_cb(struct mlx5_list *list,
-			  struct mlx5_list_entry *entry)
+flow_dv_port_id_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_port_id_action_resource *resource =
 				  container_of(entry, typeof(*resource), entry);
 
@@ -13787,10 +13773,9 @@ flow_dv_shared_rss_action_release(struct rte_eth_dev *dev, uint32_t srss)
 }
 
 void
-flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
-			    struct mlx5_list_entry *entry)
+flow_dv_push_vlan_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_push_vlan_action_resource *resource =
 			container_of(entry, typeof(*resource), entry);
 
@@ -13859,7 +13844,7 @@ flow_dv_fate_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_sample_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_sample_remove_cb(void *tool_ctx __rte_unused,
 			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_sample_resource *resource = container_of(entry,
@@ -13907,7 +13892,7 @@ flow_dv_sample_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_dest_array_remove_cb(struct mlx5_list *list __rte_unused,
+flow_dv_dest_array_remove_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_dest_array_resource *resource =
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 5450ddd388..3f2b99fb65 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index aa9e973d10..7893b3edd4 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2093,25 +2093,10 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Match an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   0 if match, none zero if not match.
- */
 int
-mlx5_hrxq_match_cb(struct mlx5_list *list,
-		   struct mlx5_list_entry *entry,
-		   void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2251,10 +2236,9 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
 	__mlx5_hrxq_remove(dev, hrxq);
@@ -2305,25 +2289,10 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
 	return NULL;
 }
 
-/**
- * Create an Rx Hash queue.
- *
- * @param list
- *   mlx5 list pointer.
- * @param entry
- *   Hash queue entry pointer.
- * @param cb_ctx
- *   Context of the callback function.
- *
- * @return
- *   queue entry on success, NULL otherwise.
- */
 struct mlx5_list_entry *
-mlx5_hrxq_create_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry __rte_unused,
-		    void *cb_ctx)
+mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq;
@@ -2333,11 +2302,10 @@ mlx5_hrxq_create_cb(struct mlx5_list *list,
 }
 
 struct mlx5_list_entry *
-mlx5_hrxq_clone_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry,
+mlx5_hrxq_clone_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		    void *cb_ctx __rte_unused)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
 	uint32_t hrxq_idx = 0;
@@ -2351,10 +2319,9 @@ mlx5_hrxq_clone_cb(struct mlx5_list *list,
 }
 
 void
-mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
-		    struct mlx5_list_entry *entry)
+mlx5_hrxq_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
 
-- 
2.25.1

[dpdk-dev] [PATCH v6 16/26] common/mlx5: add per-lcore cache to hash list utility

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates
all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and
hlist.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 238 +++++++------------
 drivers/common/mlx5/mlx5_common_utils.h | 141 +++---------
 drivers/net/mlx5/linux/mlx5_os.c        |  35 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++++++----
 drivers/net/mlx5/mlx5_flow.h            |  92 ++++----
 drivers/net/mlx5/mlx5_flow_dv.c         | 292 +++++++++++++++---------
 8 files changed, 482 insertions(+), 482 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 0f400f6353..ab098b13fa 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -13,25 +13,21 @@
 
 /********************* mlx5 list ************************/
 
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx, bool lcores_share,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
+static int
+mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
+	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       mlx5_list_match_cb cb_match,
+	       mlx5_list_remove_cb cb_remove,
+	       mlx5_list_clone_cb cb_clone,
+	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	struct mlx5_list *list;
 	int i;
 
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
-		return NULL;
+		return -EINVAL;
 	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -45,6 +41,28 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
+	return 0;
+}
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (mlx5_list_init(list, name, ctx, lcores_share,
+			   cb_create, cb_match, cb_remove, cb_clone,
+			   cb_clone_free) != 0) {
+		mlx5_free(list);
+		return NULL;
+	}
 	return list;
 }
 
@@ -254,8 +272,8 @@ mlx5_list_unregister(struct mlx5_list *list,
 	return 1;
 }
 
-void
-mlx5_list_destroy(struct mlx5_list *list)
+static void
+mlx5_list_uninit(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
 	int i;
@@ -266,15 +284,21 @@ mlx5_list_destroy(struct mlx5_list *list)
 			entry = LIST_FIRST(&list->cache[i].h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", list->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			}
 		}
 	}
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	mlx5_list_uninit(list);
 	mlx5_free(list);
 }
 
@@ -287,37 +311,24 @@ mlx5_list_get_entry_num(struct mlx5_list *list)
 
 /********************* Hash List **********************/
 
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
 struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
+		  bool lcores_share, void *ctx, mlx5_list_create_cb cb_create,
+		  mlx5_list_match_cb cb_match,
+		  mlx5_list_remove_cb cb_remove,
+		  mlx5_list_clone_cb cb_clone,
+		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+		DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
+			"be aligned to 0x%" PRIX32 ".", size, act_size);
 	} else {
 		act_size = size;
 	}
@@ -331,61 +342,24 @@ mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 			name ? name : "None");
 		return NULL;
 	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
 	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
+	h->lcores_share = lcores_share;
+	h->direct_key = direct_key;
+	for (i = 0; i < act_size; i++) {
+		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   cb_create, cb_match, cb_remove, cb_clone,
+				   cb_clone_free) != 0) {
+			mlx5_free(h);
+			return NULL;
 		}
 	}
-	return node;
+	DRV_LOG(DEBUG, "Hash list %s with size 0x%" PRIX32 " was created.",
+		name, act_size);
+	return h;
 }
 
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
 
-struct mlx5_hlist_entry *
+struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
@@ -394,102 +368,44 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
+	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
 }
 
-struct mlx5_hlist_entry*
+struct mlx5_list_entry*
 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry;
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
+	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	if (likely(entry)) {
+		if (h->lcores_share)
+			entry->gentry->bucket_idx = idx;
+		else
+			entry->bucket_idx = idx;
 	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
 	return entry;
 }
 
 int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
+	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+							      entry->bucket_idx;
+
+	return mlx5_list_unregister(&h->buckets[idx].l, entry);
 }
 
 void
 mlx5_hlist_destroy(struct mlx5_hlist *h)
 {
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
+	uint32_t i;
 
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
+	for (i = 0; i <= h->mask; i++)
+		mlx5_list_uninit(&h->buckets[i].l);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index a691ff8f0d..4bb974fa3e 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -20,10 +20,13 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t ref_cnt __rte_aligned(8); /* 0 means, entry is invalid. */
 	uint32_t lcore_idx;
-	struct mlx5_list_entry *gentry;
-};
+	union {
+		struct mlx5_list_entry *gentry;
+		uint32_t bucket_idx;
+	};
+} __rte_packed;
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
@@ -212,108 +215,24 @@ __rte_internal
 uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list);
 
-/************************ Hash list *****************************/
-
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
-
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE			32
-
-struct mlx5_hlist;
-
-/**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
- */
-struct mlx5_hlist_entry {
-	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint32_t idx; /* Bucket index the entry belongs to. */
-	uint32_t ref_cnt; /* Reference count. */
-};
-
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
-				     struct mlx5_hlist_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- * @param key
- *   The new entry key.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry,
-				   uint64_t key, void *ctx);
-
-/**
- * Type of function for user defined hash list entry creation.
- *
- * @param list
- *   The hash list.
- * @param key
- *   The key of the new entry.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer to allocated entry on success, NULL otherwise.
- */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
-				  (struct mlx5_hlist *list,
-				   uint64_t key, void *ctx);
+/********************* Hash List **********************/
 
-/* Hash list bucket head. */
+/* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_hlist_head head; /* List head. */
-	rte_rwlock_t lock; /* Bucket lock. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
+	struct mlx5_list l;
 } __rte_cache_aligned;
 
 /**
  * Hash list table structure
  *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
+ * The hash list bucket using the mlx5_list object for managing.
  */
 struct mlx5_hlist {
-	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
-	/**< number of heads, need to be power of 2. */
-	uint32_t table_sz;
-	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
-	/**< mask to get the index of the list heads. */
-	uint32_t mask;
-	bool direct_key; /* Use the new entry key directly as hash index. */
-	bool write_most; /* List mostly used for append new or destroy. */
-	void *ctx;
-	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
-	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
-	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+	uint32_t mask; /* A mask for the bucket index range. */
+	uint8_t flags;
+	bool direct_key; /* Whether to use the key directly as hash index. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
-	/**< list bucket arrays. */
 };
 
 /**
@@ -330,23 +249,33 @@ struct mlx5_hlist {
  *   Heads array size of the hash list.
  * @param entry_size
  *   Entry size to allocate if cb_create not specified.
- * @param flags
- *   The hash list attribute flags.
+ * @param direct key
+ *   Whether to use the key directly as hash index.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
+ * @param ctx
+ *   The hlist instance context.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
  *   Callback function for entry match.
- * @param cb_destroy
- *   Callback function for entry destroy.
+ * @param cb_remove
+ *   Callback function for entry remove.
+ * @param cb_clone
+ *   Callback function for entry clone.
+ * @param cb_clone_free
+ *   Callback function for entry clone free.
  * @return
  *   Pointer of the hash list table created, NULL on failure.
  */
 __rte_internal
 struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
-				     uint32_t entry_size, uint32_t flags,
-				     mlx5_hlist_create_cb cb_create,
-				     mlx5_hlist_match_cb cb_match,
-				     mlx5_hlist_remove_cb cb_destroy);
+				     bool direct_key, bool lcores_share,
+				     void *ctx, mlx5_list_create_cb cb_create,
+				     mlx5_list_match_cb cb_match,
+				     mlx5_list_remove_cb cb_remove,
+				     mlx5_list_clone_cb cb_clone,
+				     mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
@@ -365,7 +294,7 @@ struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
  *   Pointer of the hlist entry if found, NULL otherwise.
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
 					   void *ctx);
 
 /**
@@ -384,7 +313,7 @@ struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
  *   registered entry on success, NULL otherwise
  */
 __rte_internal
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+struct mlx5_list_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
 					     void *ctx);
 
 /**
@@ -399,7 +328,7 @@ struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
  *   0 on entry removed, 1 on entry still referenced.
  */
 __rte_internal
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry);
 
 /**
  * Destroy the hash list table, all the entries already inserted into the lists
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index ce41fb34a0..b04e5f8ba6 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -261,7 +261,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+	char s[MLX5_NAME_SIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -314,44 +314,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb,
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
+					  false, sh, flow_dv_tag_create_cb,
 					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb);
+					  flow_dv_tag_remove_cb,
+					  flow_dv_tag_clone_cb,
+					  flow_dv_tag_clone_free_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, MLX5_HLIST_WRITE_MOST |
-					    MLX5_HLIST_DIRECT_KEY,
+					    true, false, sh,
 					    flow_dv_modify_create_cb,
 					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb);
+					    flow_dv_modify_remove_cb,
+					    flow_dv_modify_clone_cb,
+					    flow_dv_modify_clone_free_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, MLX5_HLIST_DIRECT_KEY |
-					      MLX5_HLIST_WRITE_MOST,
+					      true, true, sh,
 					      flow_dv_encap_decap_create_cb,
 					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb);
+					      flow_dv_encap_decap_remove_cb,
+					      flow_dv_encap_decap_clone_cb,
+					     flow_dv_encap_decap_clone_free_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -1792,15 +1792,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
-						      0, 0,
+						      false, true, eth_dev,
 						      flow_dv_mreg_create_cb,
 						      flow_dv_mreg_match_cb,
-						      flow_dv_mreg_remove_cb);
+						      flow_dv_mreg_remove_cb,
+						      flow_dv_mreg_clone_cb,
+						    flow_dv_mreg_clone_free_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
-		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index d04b5d616f..fd675c9262 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1359,20 +1359,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 88b84d749a..e9b08094a6 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index d62361362e..45e90bb245 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -4040,28 +4040,27 @@ flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
 		  uint32_t flow_idx);
 
 int
-flow_dv_mreg_match_cb(struct mlx5_hlist *list __rte_unused,
-		      struct mlx5_hlist_entry *entry,
-		      uint64_t key, void *cb_ctx __rte_unused)
+flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
+		      struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
 
-	return mcp_res->mark_id != key;
+	return mcp_res->mark_id != *(uint32_t *)(ctx->data);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
-		       void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct rte_eth_dev *dev = list->ctx;
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
 	struct rte_flow_error *error = ctx->error;
 	uint32_t idx = 0;
 	int ret;
-	uint32_t mark_id = key;
+	uint32_t mark_id = *(uint32_t *)(ctx->data);
 	struct rte_flow_attr attr = {
 		.group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
 		.ingress = 1,
@@ -4167,6 +4166,36 @@ flow_dv_mreg_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return &mcp_res->hlist_ent;
 }
 
+struct mlx5_list_entry *
+flow_dv_mreg_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx __rte_unused)
+{
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	uint32_t idx = 0;
+
+	mcp_res = mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_MCP], &idx);
+	if (!mcp_res) {
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	memcpy(mcp_res, oentry, sizeof(*mcp_res));
+	mcp_res->idx = idx;
+	return &mcp_res->hlist_ent;
+}
+
+void
+flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_flow_mreg_copy_resource *mcp_res =
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
+}
+
 /**
  * Add a flow of copying flow metadata registers in RX_CP_TBL.
  *
@@ -4197,10 +4226,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 			  struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
+		.data = &mark_id,
 	};
 
 	/* Check if already registered. */
@@ -4213,11 +4243,11 @@ flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
 }
 
 void
-flow_dv_mreg_remove_cb(struct mlx5_hlist *list, struct mlx5_hlist_entry *entry)
+flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_mreg_copy_resource *mcp_res =
-		container_of(entry, typeof(*mcp_res), hlist_ent);
-	struct rte_eth_dev *dev = list->ctx;
+			       container_of(entry, typeof(*mcp_res), hlist_ent);
+	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_priv *priv = dev->data->dev_private;
 
 	MLX5_ASSERT(mcp_res->rix_flow);
@@ -4263,14 +4293,17 @@ flow_mreg_del_copy_action(struct rte_eth_dev *dev,
 static void
 flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
 {
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check if default flow is registered. */
 	if (!priv->mreg_cp_tbl)
 		return;
-	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl,
-				  MLX5_DEFAULT_COPY_ID, NULL);
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	entry = mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx);
 	if (!entry)
 		return;
 	mlx5_hlist_unregister(priv->mreg_cp_tbl, entry);
@@ -4296,6 +4329,8 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_mreg_copy_resource *mcp_res;
+	struct mlx5_flow_cb_ctx ctx;
+	uint32_t mark_id;
 
 	/* Check whether extensive metadata feature is engaged. */
 	if (!priv->config.dv_flow_en ||
@@ -4307,9 +4342,11 @@ flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
 	 * Add default mreg copy flow may be called multiple time, but
 	 * only be called once in stop. Avoid register it twice.
 	 */
-	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, MLX5_DEFAULT_COPY_ID, NULL))
+	mark_id = MLX5_DEFAULT_COPY_ID;
+	ctx.data = &mark_id;
+	if (mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, &ctx))
 		return 0;
-	mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
+	mcp_res = flow_mreg_add_copy_action(dev, mark_id, error);
 	if (!mcp_res)
 		return -rte_errno;
 	return 0;
@@ -8606,7 +8643,7 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	union tunnel_offload_mark mbits = { .val = mark };
 	union mlx5_flow_tbl_key table_key = {
 		{
@@ -8618,16 +8655,20 @@ tunnel_mark_decode(struct rte_eth_dev *dev, uint32_t mark)
 			.is_egress = 0,
 		}
 	};
-	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &table_key.v64,
+	};
+
+	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, &ctx);
 	return he ?
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
 
 static void
-mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry)
+mlx5_flow_tunnel_grp2tbl_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
@@ -8636,26 +8677,26 @@ mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
 }
 
 static int
-mlx5_flow_tunnel_grp2tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-				  struct mlx5_hlist_entry *entry,
-				  uint64_t key, void *cb_ctx __rte_unused)
+mlx5_flow_tunnel_grp2tbl_match_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *entry, void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
 
 	return tbl.tunnel_id != tte->tunnel_id || tbl.group != tte->group;
 }
 
-static struct mlx5_hlist_entry *
-mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
-				   void *ctx __rte_unused)
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key tbl = {
-		.val = key,
+		.val = *(uint64_t *)(ctx->data),
 	};
 
 	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
@@ -8684,13 +8725,36 @@ mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list, uint64_t key,
 	return NULL;
 }
 
+static struct mlx5_list_entry *
+mlx5_flow_tunnel_grp2tbl_clone_cb(void *tool_ctx __rte_unused,
+				  struct mlx5_list_entry *oentry,
+				  void *cb_ctx __rte_unused)
+{
+	struct tunnel_tbl_entry *tte = mlx5_malloc(MLX5_MEM_SYS, sizeof(*tte),
+						   0, SOCKET_ID_ANY);
+
+	if (!tte)
+		return NULL;
+	memcpy(tte, oentry, sizeof(*tte));
+	return &tte->hash;
+}
+
+static void
+mlx5_flow_tunnel_grp2tbl_clone_free_cb(void *tool_ctx __rte_unused,
+				       struct mlx5_list_entry *entry)
+{
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	mlx5_free(tte);
+}
+
 static uint32_t
 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 				const struct mlx5_flow_tunnel *tunnel,
 				uint32_t group, uint32_t *table,
 				struct rte_flow_error *error)
 {
-	struct mlx5_hlist_entry *he;
+	struct mlx5_list_entry *he;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key key = {
 		.tunnel_id = tunnel ? tunnel->tunnel_id : 0,
@@ -8698,9 +8762,12 @@ tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 	};
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_hlist *group_hash;
+	struct mlx5_flow_cb_ctx ctx = {
+		.data = &key.val,
+	};
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_register(group_hash, key.val, NULL);
+	he = mlx5_hlist_register(group_hash, key.val, &ctx);
 	if (!he)
 		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
@@ -8814,15 +8881,17 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
-					   mlx5_flow_tunnel_grp2tbl_remove_cb);
+					   mlx5_flow_tunnel_grp2tbl_remove_cb,
+					   mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(ipool, id);
 		return NULL;
 	}
-	tunnel->groups->ctx = priv->sh;
 	/* initiate new PMD tunnel */
 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
 	tunnel->tunnel_id = id;
@@ -8922,15 +8991,17 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES), 0,
-					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
+					 rte_align32pow2(MLX5_MAX_TABLES),
+					 false, true, sh,
+					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-					 mlx5_flow_tunnel_grp2tbl_remove_cb);
+					 mlx5_flow_tunnel_grp2tbl_remove_cb,
+					 mlx5_flow_tunnel_grp2tbl_clone_cb,
+					mlx5_flow_tunnel_grp2tbl_clone_free_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
 	}
-	thub->groups->ctx = sh;
 	sh->tunnel_hub = thub;
 
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 70b3119d09..e3a29297ba 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -481,7 +481,7 @@ struct mlx5_flow_dv_matcher {
 
 /* Encap/decap resource structure. */
 struct mlx5_flow_dv_encap_decap_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/* Pointer to next element. */
 	uint32_t refcnt; /**< Reference counter. */
 	void *action;
@@ -496,7 +496,7 @@ struct mlx5_flow_dv_encap_decap_resource {
 
 /* Tag resource structure. */
 struct mlx5_flow_dv_tag_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry for tag resource, tag value as the key. */
 	void *action;
 	/**< Tag action object. */
@@ -520,7 +520,7 @@ struct mlx5_flow_dv_tag_resource {
 
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
@@ -570,7 +570,7 @@ struct mlx5_flow_mreg_copy_resource {
 	 *  - Key is 32/64-bit MARK action ID.
 	 *  - MUST be the first entry.
 	 */
-	struct mlx5_hlist_entry hlist_ent;
+	struct mlx5_list_entry hlist_ent;
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
 	uint32_t idx;
@@ -587,7 +587,7 @@ struct mlx5_flow_tbl_tunnel_prm {
 
 /* Table data structure of the hash organization. */
 struct mlx5_flow_tbl_data_entry {
-	struct mlx5_hlist_entry entry;
+	struct mlx5_list_entry entry;
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
@@ -927,7 +927,7 @@ struct mlx5_flow_tunnel_hub {
 
 /* convert jump group to flow table ID in tunnel rules */
 struct tunnel_tbl_entry {
-	struct mlx5_hlist_entry hash;
+	struct mlx5_list_entry hash;
 	uint32_t flow_table;
 	uint32_t tunnel_id;
 	uint32_t group;
@@ -1585,51 +1585,59 @@ int mlx5_action_handle_flush(struct rte_eth_dev *dev);
 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id);
 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh);
 
-/* Hash list callbacks for flow tables: */
-struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *entry_ctx);
-int flow_dv_tbl_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tbl_create_cb(void *tool_ctx, void *entry_ctx);
+int flow_dv_tbl_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
+void flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tbl_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *entry_ctx);
+void flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
 struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 		uint32_t table_level, uint8_t egress, uint8_t transfer,
 		bool external, const struct mlx5_flow_tunnel *tunnel,
 		uint32_t group_id, uint8_t dummy,
 		uint32_t table_id, struct rte_flow_error *error);
 
-struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
-					       uint64_t key, void *cb_ctx);
-int flow_dv_tag_match_cb(struct mlx5_hlist *list,
-			 struct mlx5_hlist_entry *entry, uint64_t key,
+struct mlx5_list_entry *flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx);
+int flow_dv_tag_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			 void *cb_ctx);
-void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-			   struct mlx5_hlist_entry *entry);
-
-int flow_dv_modify_match_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry,
-			    uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
-						  uint64_t key, void *ctx);
-void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry);
-
-struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
-						uint64_t key, void *ctx);
-int flow_dv_mreg_match_cb(struct mlx5_hlist *list,
-			  struct mlx5_hlist_entry *entry, uint64_t key,
+void flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_tag_clone_cb(void *tool_ctx,
+					     struct mlx5_list_entry *oentry,
+					     void *cb_ctx);
+void flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_modify_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+			    void *cb_ctx);
+struct mlx5_list_entry *flow_dv_modify_create_cb(void *tool_ctx, void *ctx);
+void flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_modify_clone_cb(void *tool_ctx,
+						struct mlx5_list_entry *oentry,
+						void *ctx);
+void flow_dv_modify_clone_free_cb(void *tool_ctx,
+				  struct mlx5_list_entry *entry);
+
+struct mlx5_list_entry *flow_dv_mreg_create_cb(void *tool_ctx, void *ctx);
+int flow_dv_mreg_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			  void *cb_ctx);
-void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
-			    struct mlx5_hlist_entry *entry);
-
-int flow_dv_encap_decap_match_cb(struct mlx5_hlist *list,
-				 struct mlx5_hlist_entry *entry,
-				 uint64_t key, void *cb_ctx);
-struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-				uint64_t key, void *cb_ctx);
-void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry);
+void flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_mreg_clone_cb(void *tool_ctx,
+					      struct mlx5_list_entry *entry,
+					      void *ctx);
+void flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+
+int flow_dv_encap_decap_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+				 void *cb_ctx);
+struct mlx5_list_entry *flow_dv_encap_decap_create_cb(void *tool_ctx,
+						      void *cb_ctx);
+void flow_dv_encap_decap_remove_cb(void *tool_ctx,
+				   struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_encap_decap_clone_cb(void *tool_ctx,
+						  struct mlx5_list_entry *entry,
+						  void *cb_ctx);
+void flow_dv_encap_decap_clone_free_cb(void *tool_ctx,
+				       struct mlx5_list_entry *entry);
 
 int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 			     void *ctx);
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 795de65089..4b9fd22824 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3586,25 +3586,9 @@ flow_dv_validate_action_aso_ct(struct rte_eth_dev *dev,
 	return 0;
 }
 
-/**
- * Match encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
 int
-flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
-			     struct mlx5_hlist_entry *entry,
-			     uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3623,25 +3607,10 @@ flow_dv_encap_decap_match_cb(struct mlx5_hlist *list __rte_unused,
 	return -1;
 }
 
-/**
- * Allocate encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param ctx_cb
- *   Pointer to new encap_decap resource.
- *
- * @return
- *   0 on matching, none-zero otherwise.
- */
-struct mlx5_hlist_entry *
-flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
-			      uint64_t key __rte_unused,
-			      void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *domain;
 	struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
@@ -3679,6 +3648,38 @@ flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 	return &resource->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			     void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+	uint32_t idx;
+
+	cache_resource = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+					   &idx);
+	if (!cache_resource) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(cache_resource, oentry, sizeof(*cache_resource));
+	cache_resource->idx = idx;
+	return &cache_resource->entry;
+}
+
+void
+flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_encap_decap_resource *res =
+				       container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
+}
+
 /**
  * Find existing encap/decap resource or create and register a new one.
  *
@@ -3703,7 +3704,7 @@ flow_dv_encap_decap_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	union {
 		struct {
 			uint32_t ft_type:8;
@@ -5328,30 +5329,14 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
 	return ret;
 }
 
-/**
- * Match modify-header resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- * @param key
- *   Key of the new entry.
- * @param ctx
- *   Pointer to new modify-header resource.
- *
- * @return
- *   0 on matching, non-zero otherwise.
- */
 int
-flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
-			struct mlx5_hlist_entry *entry,
-			uint64_t key __rte_unused, void *cb_ctx)
+flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *entry, void *cb_ctx)
 {
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	struct mlx5_flow_dv_modify_hdr_resource *resource =
-			container_of(entry, typeof(*resource), entry);
+				  container_of(entry, typeof(*resource), entry);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
 
 	key_len += ref->actions_num * sizeof(ref->actions[0]);
@@ -5359,11 +5344,10 @@ flow_dv_modify_match_cb(struct mlx5_hlist *list __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
-			 void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
@@ -5402,6 +5386,33 @@ flow_dv_modify_create_cb(struct mlx5_hlist *list, uint64_t key __rte_unused,
 	return &entry->entry;
 }
 
+struct mlx5_list_entry *
+flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
+			struct mlx5_list_entry *oentry, void *cb_ctx)
+{
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *entry;
+	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+
+	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	return &entry->entry;
+}
+
+void
+flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
+			     struct mlx5_list_entry *entry)
+{
+	mlx5_free(entry);
+}
+
 /**
  * Validate the sample action.
  *
@@ -5673,7 +5684,7 @@ flow_dv_modify_hdr_resource_register
 	uint32_t key_len = sizeof(*resource) -
 			   offsetof(typeof(*resource), ft_type) +
 			   resource->actions_num * sizeof(resource->actions[0]);
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.error = error,
 		.data = resource,
@@ -9982,16 +9993,16 @@ flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
 	mlx5_free(entry);
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
+struct mlx5_list_entry *
+flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct rte_eth_dev *dev = ctx->dev;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
-	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data;
+	struct mlx5_flow_tbl_tunnel_prm *tt_prm = ctx->data2;
 	struct rte_flow_error *error = ctx->error;
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 = *(uint64_t *)(ctx->data) };
 	struct mlx5_flow_tbl_resource *tbl;
 	void *domain;
 	uint32_t idx = 0;
@@ -10068,13 +10079,13 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 }
 
 int
-flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key64,
-		     void *cb_ctx __rte_unused)
+flow_dv_tbl_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
 		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
-	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	union mlx5_flow_tbl_key key = { .v64 =  *(uint64_t *)(ctx->data) };
 
 	return tbl_data->level != key.level ||
 	       tbl_data->id != key.id ||
@@ -10083,6 +10094,39 @@ flow_dv_tbl_match_cb(struct mlx5_hlist *list __rte_unused,
 	       tbl_data->is_egress != !!key.is_egress;
 }
 
+struct mlx5_list_entry *
+flow_dv_tbl_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		      void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data;
+	struct rte_flow_error *error = ctx->error;
+	uint32_t idx = 0;
+
+	tbl_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
+	if (!tbl_data) {
+		rte_flow_error_set(error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL,
+				   "cannot allocate flow table data entry");
+		return NULL;
+	}
+	memcpy(tbl_data, oentry, sizeof(*tbl_data));
+	tbl_data->idx = idx;
+	return &tbl_data->entry;
+}
+
+void
+flow_dv_tbl_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data =
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
+}
+
 /**
  * Get a flow table.
  *
@@ -10133,9 +10177,10 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
 		.error = error,
-		.data = &tt_prm,
+		.data = &table_key.v64,
+		.data2 = &tt_prm,
 	};
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 
 	entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, &ctx);
@@ -10154,12 +10199,11 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data =
-		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+		    container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
 
 	MLX5_ASSERT(entry && sh);
 	if (tbl_data->jump.action)
@@ -10167,7 +10211,7 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 	if (tbl_data->tbl.obj)
 		mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
 	if (tbl_data->tunnel_offload && tbl_data->external) {
-		struct mlx5_hlist_entry *he;
+		struct mlx5_list_entry *he;
 		struct mlx5_hlist *tunnel_grp_hash;
 		struct mlx5_flow_tunnel_hub *thub = sh->tunnel_hub;
 		union tunnel_tbl_key tunnel_key = {
@@ -10176,11 +10220,14 @@ flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
 			.group = tbl_data->group_id
 		};
 		uint32_t table_level = tbl_data->level;
+		struct mlx5_flow_cb_ctx ctx = {
+			.data = (void *)&tunnel_key.val,
+		};
 
 		tunnel_grp_hash = tbl_data->tunnel ?
 					tbl_data->tunnel->groups :
 					thub->groups;
-		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
+		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, &ctx);
 		if (he)
 			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
@@ -10333,29 +10380,29 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
 	return 0;
 }
 
-struct mlx5_hlist_entry *
-flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
+struct mlx5_list_entry *
+flow_dv_tag_create_cb(void *tool_ctx, void *cb_ctx)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
-	struct rte_flow_error *error = ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_tag_resource *entry;
 	uint32_t idx = 0;
 	int ret;
 
 	entry = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
 	if (!entry) {
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "cannot allocate resource memory");
 		return NULL;
 	}
 	entry->idx = idx;
-	entry->tag_id = key;
-	ret = mlx5_flow_os_create_flow_action_tag(key,
+	entry->tag_id = *(uint32_t *)(ctx->data);
+	ret = mlx5_flow_os_create_flow_action_tag(entry->tag_id,
 						  &entry->action);
 	if (ret) {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], idx);
-		rte_flow_error_set(error, ENOMEM,
+		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create action");
 		return NULL;
@@ -10364,14 +10411,45 @@ flow_dv_tag_create_cb(struct mlx5_hlist *list, uint64_t key, void *ctx)
 }
 
 int
-flow_dv_tag_match_cb(struct mlx5_hlist *list __rte_unused,
-		     struct mlx5_hlist_entry *entry, uint64_t key,
-		     void *cb_ctx __rte_unused)
+flow_dv_tag_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		     void *cb_ctx)
 {
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
-	return key != tag->tag_id;
+	return *(uint32_t *)(ctx->data) != tag->tag_id;
+}
+
+struct mlx5_list_entry *
+flow_dv_tag_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+		     void *cb_ctx)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
+	struct mlx5_flow_dv_tag_resource *entry;
+	uint32_t idx = 0;
+
+	entry = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TAG], &idx);
+	if (!entry) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+				   "cannot allocate tag resource memory");
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry));
+	entry->idx = idx;
+	return &entry->entry;
+}
+
+void
+flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_tag_resource *tag =
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TAG], tag->idx);
 }
 
 /**
@@ -10398,9 +10476,13 @@ flow_dv_tag_resource_register
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_dv_tag_resource *resource;
-	struct mlx5_hlist_entry *entry;
+	struct mlx5_list_entry *entry;
+	struct mlx5_flow_cb_ctx ctx = {
+					.error = error,
+					.data = &tag_be24,
+					};
 
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, error);
+	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
@@ -10412,12 +10494,11 @@ flow_dv_tag_resource_register
 }
 
 void
-flow_dv_tag_remove_cb(struct mlx5_hlist *list,
-		      struct mlx5_hlist_entry *entry)
+flow_dv_tag_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_tag_resource *tag =
-		container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
+		   container_of(entry, struct mlx5_flow_dv_tag_resource, entry);
 
 	MLX5_ASSERT(tag && sh && tag->action);
 	claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
@@ -13612,19 +13693,10 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 	return ret;
 }
 
-/**
- * Release encap_decap resource.
- *
- * @param list
- *   Pointer to the hash list.
- * @param entry
- *   Pointer to exist resource entry object.
- */
 void
-flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
-			      struct mlx5_hlist_entry *entry)
+flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_dv_encap_decap_resource *res =
 				       container_of(entry, typeof(*res), entry);
 
@@ -13684,8 +13756,8 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(struct mlx5_hlist *list __rte_unused,
-			 struct mlx5_hlist_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
-- 
2.25.1

[dpdk-dev] [PATCH v6 17/26] common/mlx5: allocate cache list memory individually

[Suanming Mou]

Currently, the list's local cache instance memory is allocated with
the list. As the local cache instance array size is RTE_MAX_LCORE,
most of the cases the system will only have very limited cores.
allocate the instance memory individually per core will be more
economic to the memory.

This commit changes the instance array to pointer array, allocate
the local cache memory only when the core is to be used.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index ab098b13fa..18d38355fa 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 4bb974fa3e..d49fb64457 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-- 
2.25.1

[dpdk-dev] [PATCH v6 18/26] common/mlx5: optimize cache list object memory

[Suanming Mou]

Currently, hash list uses the cache list as bucket list. The list
in the buckets have the same name, ctx and callbacks. This wastes
the memory.

This commit abstracts all the name, ctx and callback members in the
list to a constant struct and others to the inconstant struct, uses
the wrapper functions to satisfy both hash list and cache list can
set the list constant and inconstant struct individually.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 18d38355fa..573e40e88c 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index d49fb64457..5718a21be0 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -80,6 +80,32 @@ typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
 typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
 						       void *ctx);
 
+/**
+ * Linked mlx5 list constant object.
+ */
+struct mlx5_list_const {
+	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
+	void *ctx; /* user objects target to callback. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
+	mlx5_list_create_cb cb_create; /**< entry create callback. */
+	mlx5_list_match_cb cb_match; /**< entry match callback. */
+	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	/**< entry clone free callback. */
+};
+
+/**
+ * Linked mlx5 list inconstant data.
+ */
+struct mlx5_list_inconst {
+	rte_rwlock_t lock; /* read/write lock. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
+};
+
 /**
  * Linked mlx5 list structure.
  *
@@ -96,19 +122,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
  *
  */
 struct mlx5_list {
-	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	void *ctx; /* user objects target to callback. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
-	mlx5_list_create_cb cb_create; /**< entry create callback. */
-	mlx5_list_match_cb cb_match; /**< entry match callback. */
-	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
-	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
-	/* Lcore cache, last index is the global cache. */
-	volatile uint32_t gen_cnt; /* List modification may update it. */
-	volatile uint32_t count; /* number of entries in list. */
-	rte_rwlock_t lock; /* read/write lock. */
+	struct mlx5_list_const l_const;
+	struct mlx5_list_inconst l_inconst;
 };
 
 /**
@@ -219,7 +234,7 @@ mlx5_list_get_entry_num(struct mlx5_list *list);
 
 /* Hash list bucket. */
 struct mlx5_hlist_bucket {
-	struct mlx5_list l;
+	struct mlx5_list_inconst l;
 } __rte_cache_aligned;
 
 /**
@@ -231,7 +246,7 @@ struct mlx5_hlist {
 	uint32_t mask; /* A mask for the bucket index range. */
 	uint8_t flags;
 	bool direct_key; /* Whether to use the key directly as hash index. */
-	bool lcores_share; /* Whether to share objects between the lcores. */
+	struct mlx5_list_const l_const; /* List constant data. */
 	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v6 19/26] common/mlx5: support list non-lcore operations

[Suanming Mou]

This commit supports the list non-lcore operations with
an extra sub-list and lock.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
index 573e40e88c..fcc81f05c6 100644
--- a/drivers/common/mlx5/mlx5_common_utils.c
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.lcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry = NULL, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.lcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
+	return ret;
 }
 
 void
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 5718a21be0..613d29de0c 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t lcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
-- 
2.25.1

[dpdk-dev] [PATCH v6 20/26] net/mlx5: move modify header allocator to ipool

[Suanming Mou]

From: Matan Azrad <matan@nvidia.com>

Modify header actions are allocated by mlx5_malloc which has a big
overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has
its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in
the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has
better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header
handler.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index fd675c9262..640d36c6be 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -802,6 +802,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -815,6 +816,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e9b08094a6..e7e4749824 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1146,6 +1159,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index e3a29297ba..7027012220 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -505,23 +505,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 4b9fd22824..e39fe43854 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5344,6 +5344,45 @@ flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
 	       memcmp(&ref->ft_type, &resource->ft_type, key_len);
 }
 
+static struct mlx5_indexed_pool *
+flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
+{
+	struct mlx5_indexed_pool *ipool = __atomic_load_n
+				     (&sh->mdh_ipools[index], __ATOMIC_SEQ_CST);
+
+	if (!ipool) {
+		struct mlx5_indexed_pool *expected = NULL;
+		struct mlx5_indexed_pool_config cfg =
+		    (struct mlx5_indexed_pool_config) {
+		       .size = sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
+								   (index + 1) *
+					   sizeof(struct mlx5_modification_cmd),
+		       .trunk_size = 64,
+		       .grow_trunk = 3,
+		       .grow_shift = 2,
+		       .need_lock = 1,
+		       .release_mem_en = 1,
+		       .malloc = mlx5_malloc,
+		       .free = mlx5_free,
+		       .type = "mlx5_modify_action_resource",
+		};
+
+		cfg.size = RTE_ALIGN(cfg.size, sizeof(ipool));
+		ipool = mlx5_ipool_create(&cfg);
+		if (!ipool)
+			return NULL;
+		if (!__atomic_compare_exchange_n(&sh->mdh_ipools[index],
+						 &expected, ipool, false,
+						 __ATOMIC_SEQ_CST,
+						 __ATOMIC_SEQ_CST)) {
+			mlx5_ipool_destroy(ipool);
+			ipool = __atomic_load_n(&sh->mdh_ipools[index],
+						__ATOMIC_SEQ_CST);
+		}
+	}
+	return ipool;
+}
+
 struct mlx5_list_entry *
 flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 {
@@ -5352,12 +5391,20 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 	struct mlx5dv_dr_domain *ns;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
+	struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
+							  ref->actions_num - 1);
 	int ret;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
 	uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
+	uint32_t idx;
 
-	entry = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*entry) + data_len, 0,
-			    SOCKET_ID_ANY);
+	if (unlikely(!ipool)) {
+		rte_flow_error_set(ctx->error, ENOMEM,
+				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				   NULL, "cannot allocate modify ipool");
+		return NULL;
+	}
+	entry = mlx5_ipool_zmalloc(ipool, &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5377,25 +5424,29 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
 					(sh->ctx, ns, entry,
 					 data_len, &entry->action);
 	if (ret) {
-		mlx5_free(entry);
+		mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				   NULL, "cannot create modification action");
 		return NULL;
 	}
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 struct mlx5_list_entry *
-flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
-			struct mlx5_list_entry *oentry, void *cb_ctx)
+flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+			void *cb_ctx)
 {
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_dv_modify_hdr_resource *entry;
 	struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
 	uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
+	uint32_t idx;
 
-	entry = mlx5_malloc(0, sizeof(*entry) + data_len, 0, SOCKET_ID_ANY);
+	entry = mlx5_ipool_malloc(sh->mdh_ipools[ref->actions_num - 1],
+				  &idx);
 	if (!entry) {
 		rte_flow_error_set(ctx->error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -5403,14 +5454,18 @@ flow_dv_modify_clone_cb(void *tool_ctx __rte_unused,
 		return NULL;
 	}
 	memcpy(entry, oentry, sizeof(*entry) + data_len);
+	entry->idx = idx;
 	return &entry->entry;
 }
 
 void
-flow_dv_modify_clone_free_cb(void *tool_ctx __rte_unused,
-			     struct mlx5_list_entry *entry)
+flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
-	mlx5_free(entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
+	struct mlx5_flow_dv_modify_hdr_resource *res =
+		container_of(entry, typeof(*res), entry);
+
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
@@ -13756,14 +13811,14 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
 }
 
 void
-flow_dv_modify_remove_cb(void *tool_ctx __rte_unused,
-			 struct mlx5_list_entry *entry)
+flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 {
 	struct mlx5_flow_dv_modify_hdr_resource *res =
 		container_of(entry, typeof(*res), entry);
+	struct mlx5_dev_ctx_shared *sh = tool_ctx;
 
 	claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
-	mlx5_free(entry);
+	mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v6 21/26] net/mlx5: adjust the hash bucket size

[Suanming Mou]

With the new per core optimization to the list, the hash bucket size
can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index b04e5f8ba6..969d265350 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 640d36c6be..3b5c7840a8 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -376,7 +376,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 906aa43c5a..ca67ce8213 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 45e90bb245..1c5d4c283a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -8881,7 +8881,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8990,8 +8990,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
-- 
2.25.1

[dpdk-dev] [PATCH v6 22/26] net/mlx5: enable index pool per-core cache

[Suanming Mou]

This commit enables the tag and header modify action index pool
per-core cache in non-reclaim memory mode.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 3b5c7840a8..f79a094a35 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -217,7 +217,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1129,6 +1130,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e7e4749824..30893c7f38 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1116,6 +1116,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index e39fe43854..739c387c0a 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -5361,7 +5361,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
-- 
2.25.1

[dpdk-dev] [PATCH v6 23/26] net/mlx5: optimize hash list table allocate on demand

[Suanming Mou]

Currently, all the hash list tables are allocated during start up.
Since different applications may only use dedicated limited actions,
optimized the hash list table allocate on demand will save initial
memory.

This commit optimizes hash list table allocate on demand.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 969d265350..ec2f8a4aed 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index ca67ce8213..fe86bb40d3 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 739c387c0a..5b8eb096ee 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -311,6 +311,41 @@ mlx5_flow_tunnel_ip_check(const struct rte_flow_item *item __rte_unused,
 	}
 }
 
+static inline struct mlx5_hlist *
+flow_dv_hlist_prepare(struct mlx5_dev_ctx_shared *sh, struct mlx5_hlist **phl,
+		     const char *name, uint32_t size, bool direct_key,
+		     bool lcores_share, void *ctx,
+		     mlx5_list_create_cb cb_create,
+		     mlx5_list_match_cb cb_match,
+		     mlx5_list_remove_cb cb_remove,
+		     mlx5_list_clone_cb cb_clone,
+		     mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_hlist *hl;
+	struct mlx5_hlist *expected = NULL;
+	char s[MLX5_NAME_SIZE];
+
+	hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	if (likely(hl))
+		return hl;
+	snprintf(s, sizeof(s), "%s_%s", sh->ibdev_name, name);
+	hl = mlx5_hlist_create(s, size, direct_key, lcores_share,
+			ctx, cb_create, cb_match, cb_remove, cb_clone,
+			cb_clone_free);
+	if (!hl) {
+		DRV_LOG(ERR, "%s hash creation failed", name);
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	if (!__atomic_compare_exchange_n(phl, &expected, hl, false,
+					 __ATOMIC_SEQ_CST,
+					 __ATOMIC_SEQ_CST)) {
+		mlx5_hlist_destroy(hl);
+		hl = __atomic_load_n(phl, __ATOMIC_SEQ_CST);
+	}
+	return hl;
+}
+
 /* Update VLAN's VID/PCP based on input rte_flow_action.
  *
  * @param[in] action
@@ -3730,8 +3765,20 @@ flow_dv_encap_decap_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *encaps_decaps;
 	uint64_t key64;
 
+	encaps_decaps = flow_dv_hlist_prepare(sh, &sh->encaps_decaps,
+				"encaps_decaps",
+				MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
+				true, true, sh,
+				flow_dv_encap_decap_create_cb,
+				flow_dv_encap_decap_match_cb,
+				flow_dv_encap_decap_remove_cb,
+				flow_dv_encap_decap_clone_cb,
+				flow_dv_encap_decap_clone_free_cb);
+	if (unlikely(!encaps_decaps))
+		return -rte_errno;
 	resource->flags = dev_flow->dv.group ? 0 : 1;
 	key64 =  __rte_raw_cksum(&encap_decap_key.v32,
 				 sizeof(encap_decap_key.v32), 0);
@@ -3739,7 +3786,7 @@ flow_dv_encap_decap_resource_register
 	    MLX5DV_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 &&
 	    resource->size)
 		key64 = __rte_raw_cksum(resource->buf, resource->size, key64);
-	entry = mlx5_hlist_register(sh->encaps_decaps, key64, &ctx);
+	entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -5745,8 +5792,20 @@ flow_dv_modify_hdr_resource_register
 		.error = error,
 		.data = resource,
 	};
+	struct mlx5_hlist *modify_cmds;
 	uint64_t key64;
 
+	modify_cmds = flow_dv_hlist_prepare(sh, &sh->modify_cmds,
+				"hdr_modify",
+				MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
+				true, false, sh,
+				flow_dv_modify_create_cb,
+				flow_dv_modify_match_cb,
+				flow_dv_modify_remove_cb,
+				flow_dv_modify_clone_cb,
+				flow_dv_modify_clone_free_cb);
+	if (unlikely(!modify_cmds))
+		return -rte_errno;
 	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
 								resource->root))
@@ -5754,7 +5813,7 @@ flow_dv_modify_hdr_resource_register
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
 	key64 = __rte_raw_cksum(&resource->ft_type, key_len, 0);
-	entry = mlx5_hlist_register(sh->modify_cmds, key64, &ctx);
+	entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
 	if (!entry)
 		return -rte_errno;
 	resource = container_of(entry, typeof(*resource), entry);
@@ -10537,8 +10596,20 @@ flow_dv_tag_resource_register
 					.error = error,
 					.data = &tag_be24,
 					};
-
-	entry = mlx5_hlist_register(priv->sh->tag_table, tag_be24, &ctx);
+	struct mlx5_hlist *tag_table;
+
+	tag_table = flow_dv_hlist_prepare(priv->sh, &priv->sh->tag_table,
+				      "tags",
+				      MLX5_TAGS_HLIST_ARRAY_SIZE,
+				      false, false, priv->sh,
+				      flow_dv_tag_create_cb,
+				      flow_dv_tag_match_cb,
+				      flow_dv_tag_remove_cb,
+				      flow_dv_tag_clone_cb,
+				      flow_dv_tag_clone_free_cb);
+	if (unlikely(!tag_table))
+		return -rte_errno;
+	entry = mlx5_hlist_register(tag_table, tag_be24, &ctx);
 	if (entry) {
 		resource = container_of(entry, struct mlx5_flow_dv_tag_resource,
 					entry);
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index a04f93e1d4..5da362a9d5 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
-- 
2.25.1

[dpdk-dev] [PATCH v6 24/26] net/mlx5: change memory release configuration

[Suanming Mou]

This commit changes the index pool memory release configuration
to 0 when memory reclaim mode is not required.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f79a094a35..021a34dd4d 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -798,6 +798,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
-- 
2.25.1

[dpdk-dev] [PATCH v6 25/26] net/mlx5: optimize Rx queue match

[Suanming Mou]

As hrxq struct has the indirect table pointer, while matching the
hrxq, better to use the hrxq indirect table instead of searching
from the list.

This commit optimizes the hrxq indirect table matching.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 7893b3edd4..23685d7654 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
-- 
2.25.1

[dpdk-dev] [PATCH v6 26/26] doc: add mlx5 multiple-thread flow insertion optimization

[Suanming Mou]

This commit adds the multiple-thread flow insertion optimization
description.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/nics/mlx5.rst               | 5 +++++
 doc/guides/rel_notes/release_21_08.rst | 1 +
 2 files changed, 6 insertions(+)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 5a3c4f198a..199beb2549 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -448,6 +448,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index 6a902ef9ac..68496496e8 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -88,6 +88,7 @@ New Features
 
   * Added support for meter hierarchy.
   * Added devargs options ``allow_duplicate_pattern``.
+  * Optimize multiple-thread flow insertion rate.
 
 * **Added support for Marvell CNXK crypto driver.**
 
-- 
2.25.1

Re: [dpdk-dev] [PATCH v6 00/26] net/mlx5: insertion rate optimization

[Raslan Darawsheh]

Hi,

> -----Original Message-----
> From: Suanming Mou <suanmingm@nvidia.com>
> Sent: Tuesday, July 13, 2021 11:45 AM
> To: Slava Ovsiienko <viacheslavo@nvidia.com>; Matan Azrad
> <matan@nvidia.com>
> Cc: Raslan Darawsheh <rasland@nvidia.com>; Ori Kam <orika@nvidia.com>;
> dev@dpdk.org
> Subject: [PATCH v6 00/26] net/mlx5: insertion rate optimization
> 
> This patch series optimize the flow insertion rate with adding local cache to
> index pool and list.
> 
> For object which wants efficient index allocate and free, local cache will be
> very helpful.
> 
> For index pool, two level cache is added, one as local and another as global.
> The global cache is able to save all the allocated index. That means all the
> allocated index will not be freed. Once the local cache is full, the extra index
> will be flushed to the global cache. Once local cache is empty, first try to fetch
> more index from global, if global is still empty, allocate new trunk and more
> index.
> 
> For list, sub local core list is introduced. The allocated objects will be added
> and released only from local list without any locks.
> Only the objects need to be shared will be synced from global list.
> 
> ---
> 
> v6: fix compilation issue in "common/mlx5: add list lcore share" patch.
> 
> v5: code rebase to the latest version.
> 
> v4:
>    - split the list utility to commom patch
>    - fix one ipool crash issue in mlx5_ipool_get_cache() function.
>    - reorganize the common and mlx5 patches.
>    - split the doc update to an individual patch.
>    - update and rebase to the latest version.
> 
> v3: fix Windows compilation issue
> 
> v2: add the list per-lcore cache optimization
> 
> ---
> 
> Matan Azrad (9):
>   net/mlx5: optimize modify header action memory
>   net/mlx5: remove cache term from the list utility
>   net/mlx5: add per lcore cache to the list utility
>   net/mlx5: minimize list critical sections
>   net/mlx5: manage list cache entries release
>   net/mlx5: relax the list utility atomic operations
>   net/mlx5: allocate list memory by the create API
>   common/mlx5: add per-lcore cache to hash list utility
>   net/mlx5: move modify header allocator to ipool
> 
> Suanming Mou (17):
>   net/mlx5: allow limiting the index pool maximum index
>   net/mlx5: add indexed pool local cache
>   net/mlx5: add index pool foreach define
>   net/mlx5: support index pool non-lcore operations
>   net/mlx5: replace flow list with index pool
>   common/mlx5: move list utility to common
>   common/mlx5: add list lcore share
>   common/mlx5: call list callbacks with context
>   common/mlx5: allocate cache list memory individually
>   common/mlx5: optimize cache list object memory
>   common/mlx5: support list non-lcore operations
>   net/mlx5: adjust the hash bucket size
>   net/mlx5: enable index pool per-core cache
>   net/mlx5: optimize hash list table allocate on demand
>   net/mlx5: change memory release configuration
>   net/mlx5: optimize Rx queue match
>   doc: add mlx5 multiple-thread flow insertion optimization
> 
>  doc/guides/nics/mlx5.rst                |    5 +
>  doc/guides/rel_notes/release_21_08.rst  |    1 +
>  drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
>  drivers/common/mlx5/mlx5_common.h       |    2 +
>  drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
> drivers/common/mlx5/mlx5_common_utils.h |  294 ++++--
>  drivers/common/mlx5/version.map         |    7 +
>  drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
>  drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
>  drivers/net/mlx5/mlx5.c                 |   34 +-
>  drivers/net/mlx5/mlx5.h                 |   46 +-
>  drivers/net/mlx5/mlx5_defs.h            |   12 +-
>  drivers/net/mlx5/mlx5_flow.c            |  308 +++---
>  drivers/net/mlx5/mlx5_flow.h            |  209 ++--
>  drivers/net/mlx5/mlx5_flow_dv.c         | 1206 +++++++++++++++--------
>  drivers/net/mlx5/mlx5_rx.h              |   14 +-
>  drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
>  drivers/net/mlx5/mlx5_trigger.c         |    8 +-
>  drivers/net/mlx5/mlx5_utils.c           |  627 ++++++++----
>  drivers/net/mlx5/mlx5_utils.h           |  255 ++---
>  drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
>  21 files changed, 2543 insertions(+), 1414 deletions(-)
> 
> --
> 2.25.1

Series applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh