[dpdk-dev] [PATCH v2 09/22] net/mlx5: manage list cache entries release

[Suanming Mou <suanmingm@nvidia.com>]

From: Matan Azrad <matan@nvidia.com>

When a cache entry is allocated by lcore A and is released by lcore B,
the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be
increased atomically when the non-original lcore decreases the reference
counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in
its cache, it will check the counter in order to free invalid entries in
its cache.

Signed-off-by: Matan Azrad <matan@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index 772b352af5..7cdf44dcf7 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 684d1e8a2a..ffa9cd5142 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -310,11 +310,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
-- 
2.25.1