[PATCH v3 0/7] support reinit flow

[PATCH v3 0/7] support reinit flow

[okaya]

From: Sinan Kaya <okaya@kernel.org>

We want to be able to call rte_eal_init() and rte_eal_cleanup()
APIs back to back for maintanance reasons.

Here is a summary of the code we have seen so far:

1. some code support getting called multiple times by keeping
a static variable.
2. some code initializes once but never clean up after them and
don't have a cleanup API.
3. some code assumes that they only get called once during the
lifecycle of the process.

Most changes in this patch center around following the #1 design
principle.

Why?

It is not always ideal to reinitialize a DPDK process. Memory needs
to be reinitialized, hugetables need to warm up etc.

Changed from

v1:
Fix checkpatch warnings

v2:
rebase to most recent DPDK.

Graham Whyte (1):
  eal: fixes for re-initialization issues

Sinan Kaya (6):
  tailq: skip init if already initialized
  eal_memzone: bail out on initialized
  memseg: init once
  eal_memory: skip initialization
  eal_interrupts: don't reinitialize threads
  eal: initialize worker threads once

 lib/eal/common/eal_common_memory.c  |  5 +++
 lib/eal/common/eal_common_memzone.c |  7 ++++
 lib/eal/common/eal_common_options.c |  7 ++++
 lib/eal/common/eal_common_tailqs.c  | 20 +++++----
 lib/eal/common/malloc_heap.c        |  7 ++++
 lib/eal/linux/eal.c                 | 64 ++++++++++++++++-------------
 lib/eal/linux/eal_interrupts.c      |  7 ++++
 lib/eal/linux/eal_memory.c          | 12 +++++-
 8 files changed, 92 insertions(+), 37 deletions(-)

-- 
2.25.1

[PATCH v3 1/7] eal: fixes for re-initialization issues

[okaya]

From: Graham Whyte <grwhyte@microsoft.com>

reinitialize the solib link list and clean the globals holding
state for parsing.

Signed-off-by: Sinan Kaya <okaya@kernel.org>
Signed-off-by: Graham Whyte <grwhyte@microsoft.com>
---
 lib/eal/common/eal_common_options.c | 7 +++++++
 lib/eal/linux/eal.c                 | 4 +++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/eal/common/eal_common_options.c b/lib/eal/common/eal_common_options.c
index 062f1d8d9c..7043b6e2fd 100644
--- a/lib/eal/common/eal_common_options.c
+++ b/lib/eal/common/eal_common_options.c
@@ -1981,6 +1981,13 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
 		memcpy(cpuset, &lcore_config[rte_get_main_lcore()].cpuset,
 			sizeof(*cpuset));
 	}
+
+	/* Reinitialize solib_list */
+	TAILQ_INIT(&solib_list);
+
+	main_lcore_parsed = 0;
+	mem_parsed = 0;
+	core_parsed = 0;
 }
 
 int
diff --git a/lib/eal/linux/eal.c b/lib/eal/linux/eal.c
index c6efd92014..650fb5f5a9 100644
--- a/lib/eal/linux/eal.c
+++ b/lib/eal/linux/eal.c
@@ -79,6 +79,8 @@ struct lcore_config lcore_config[RTE_MAX_LCORE];
 int rte_cycles_vmware_tsc_map;
 
 
+static uint32_t run_once;
+
 int
 eal_clean_runtime_dir(void)
 {
@@ -505,6 +507,7 @@ eal_parse_socket_arg(char *strval, volatile uint64_t *socket_arg)
 		socket_arg[i] = val;
 	}
 
+	__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
 	return 0;
 }
 
@@ -967,7 +970,6 @@ int
 rte_eal_init(int argc, char **argv)
 {
 	int i, fctret, ret;
-	static uint32_t run_once;
 	uint32_t has_run = 0;
 	char cpuset[RTE_CPU_AFFINITY_STR_LEN];
 	char thread_name[RTE_MAX_THREAD_NAME_LEN];
-- 
2.25.1

[PATCH v3 2/7] tailq: skip init if already initialized

[okaya]

From: Sinan Kaya <okaya@kernel.org>

Allows tailq to be reinitialied multiple times
by looking up previously registered tailqs

Signed-off-by: Sinan Kaya <okaya@kernel.org>
---
 lib/eal/common/eal_common_tailqs.c | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/lib/eal/common/eal_common_tailqs.c b/lib/eal/common/eal_common_tailqs.c
index 580fbf24bc..75c0235438 100644
--- a/lib/eal/common/eal_common_tailqs.c
+++ b/lib/eal/common/eal_common_tailqs.c
@@ -73,9 +73,10 @@ rte_eal_tailq_create(const char *name)
 		strlcpy(head->name, name, sizeof(head->name) - 1);
 		TAILQ_INIT(&head->tailq_head);
 		rte_tailqs_count++;
+		return head;
 	}
 
-	return head;
+	return rte_eal_tailq_lookup(name);
 }
 
 /* local register, used to store "early" tailqs before rte_eal_init() and to
@@ -99,7 +100,9 @@ rte_eal_tailq_update(struct rte_tailq_elem *t)
 {
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 		/* primary process is the only one that creates */
-		t->head = rte_eal_tailq_create(t->name);
+		t->head = rte_eal_tailq_lookup(t->name);
+		if (t->head == NULL)
+			t->head = rte_eal_tailq_create(t->name);
 	} else {
 		t->head = rte_eal_tailq_lookup(t->name);
 	}
@@ -108,15 +111,13 @@ rte_eal_tailq_update(struct rte_tailq_elem *t)
 int
 rte_eal_tailq_register(struct rte_tailq_elem *t)
 {
-	if (rte_eal_tailq_local_register(t) < 0) {
-		RTE_LOG(ERR, EAL,
-			"%s tailq is already registered\n", t->name);
-		goto error;
-	}
+	rte_eal_tailq_local_register(t);
 
 	/* if a register happens after rte_eal_tailqs_init(), then we can update
 	 * tailq head */
 	if (rte_tailqs_count >= 0) {
+		RTE_LOG(INFO, EAL,
+			"%s tailq is registered\n", t->name);
 		rte_eal_tailq_update(t);
 		if (t->head == NULL) {
 			RTE_LOG(ERR, EAL,
@@ -138,6 +139,11 @@ rte_eal_tailqs_init(void)
 {
 	struct rte_tailq_elem *t;
 
+	if (rte_tailqs_count > 0) {
+		RTE_LOG(INFO, EAL, "tailq already initialized\n");
+		return 0;
+	}
+
 	rte_tailqs_count = 0;
 
 	TAILQ_FOREACH(t, &rte_tailq_elem_head, next) {
-- 
2.25.1

[PATCH v3 3/7] eal_memzone: bail out on initialized

[okaya]

From: Sinan Kaya <okaya@kernel.org>

Initialize memzone once and bail out if someone calls init
multiple times.

Signed-off-by: Sinan Kaya <okaya@kernel.org>
---
 lib/eal/common/eal_common_memzone.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/eal/common/eal_common_memzone.c b/lib/eal/common/eal_common_memzone.c
index 1f3e701499..6645ccfe83 100644
--- a/lib/eal/common/eal_common_memzone.c
+++ b/lib/eal/common/eal_common_memzone.c
@@ -22,6 +22,8 @@
 #include "eal_private.h"
 #include "eal_memcfg.h"
 
+static bool memzone_initialized;
+
 /* Default count used until rte_memzone_max_set() is called */
 #define DEFAULT_MAX_MEMZONE_COUNT 2560
 
@@ -426,6 +428,9 @@ rte_eal_memzone_init(void)
 	struct rte_mem_config *mcfg;
 	int ret = 0;
 
+	if (memzone_initialized)
+		return 0;
+
 	/* get pointer to global configuration */
 	mcfg = rte_eal_get_configuration()->mem_config;
 
@@ -444,6 +449,8 @@ rte_eal_memzone_init(void)
 
 	rte_rwlock_write_unlock(&mcfg->mlock);
 
+	memzone_initialized = true;
+
 	return ret;
 }
 
-- 
2.25.1

[PATCH v3 4/7] memseg: init once

[okaya]

From: Sinan Kaya <okaya@kernel.org>

Initialize memory segments just once and bail out if called
multiple times.

Signed-off-by: Sinan Kaya <okaya@kernel.org>
---
 lib/eal/linux/eal_memory.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/lib/eal/linux/eal_memory.c b/lib/eal/linux/eal_memory.c
index 9b6f08fba8..df0aa9ccc7 100644
--- a/lib/eal/linux/eal_memory.c
+++ b/lib/eal/linux/eal_memory.c
@@ -57,6 +57,7 @@
  */
 
 static int phys_addrs_available = -1;
+static bool memseg_initialized;
 
 #define RANDOMIZE_VA_SPACE_FILE "/proc/sys/kernel/randomize_va_space"
 
@@ -1920,6 +1921,10 @@ rte_eal_memseg_init(void)
 {
 	/* increase rlimit to maximum */
 	struct rlimit lim;
+	int ret;
+
+	if (memseg_initialized)
+		return 0;
 
 #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
 	const struct internal_config *internal_conf =
@@ -1948,11 +1953,16 @@ rte_eal_memseg_init(void)
 	}
 #endif
 
-	return rte_eal_process_type() == RTE_PROC_PRIMARY ?
+	ret = rte_eal_process_type() == RTE_PROC_PRIMARY ?
 #ifndef RTE_ARCH_64
 			memseg_primary_init_32() :
 #else
 			memseg_primary_init() :
 #endif
 			memseg_secondary_init();
+
+	if (!ret)
+		memseg_initialized = true;
+
+	return ret;
 }
-- 
2.25.1

[PATCH v3 5/7] eal_memory: skip initialization

[okaya]

From: Sinan Kaya <okaya@kernel.org>

Initialize heap area just once.

Signed-off-by: Sinan Kaya <okaya@kernel.org>
---
 lib/eal/common/eal_common_memory.c | 5 +++++
 lib/eal/common/malloc_heap.c       | 7 +++++++
 2 files changed, 12 insertions(+)

diff --git a/lib/eal/common/eal_common_memory.c b/lib/eal/common/eal_common_memory.c
index d9433db623..4c68de1054 100644
--- a/lib/eal/common/eal_common_memory.c
+++ b/lib/eal/common/eal_common_memory.c
@@ -41,6 +41,7 @@
 
 static void *next_baseaddr;
 static uint64_t system_page_sz;
+static bool memory_initialized;
 
 #define MAX_MMAP_WITH_DEFINED_ADDR_TRIES 5
 void *
@@ -1084,6 +1085,9 @@ rte_eal_memory_init(void)
 		eal_get_internal_configuration();
 	int retval;
 
+	if (memory_initialized)
+		return 0;
+
 	RTE_LOG(DEBUG, EAL, "Setting up physically contiguous memory...\n");
 
 	if (rte_eal_memseg_init() < 0)
@@ -1101,6 +1105,7 @@ rte_eal_memory_init(void)
 	if (internal_conf->no_shconf == 0 && rte_eal_memdevice_init() < 0)
 		goto fail;
 
+	memory_initialized = true;
 	return 0;
 fail:
 	return -1;
diff --git a/lib/eal/common/malloc_heap.c b/lib/eal/common/malloc_heap.c
index 6b6cf9174c..8d05a2ab2e 100644
--- a/lib/eal/common/malloc_heap.c
+++ b/lib/eal/common/malloc_heap.c
@@ -31,6 +31,8 @@
 #define CONST_MAX(a, b) (a > b ? a : b) /* RTE_MAX is not a constant */
 #define EXTERNAL_HEAP_MIN_SOCKET_ID (CONST_MAX((1 << 8), RTE_MAX_NUMA_NODES))
 
+static bool heap_initialized;
+
 static unsigned
 check_hugepage_sz(unsigned flags, uint64_t hugepage_sz)
 {
@@ -1410,6 +1412,9 @@ rte_eal_malloc_heap_init(void)
 	const struct internal_config *internal_conf =
 		eal_get_internal_configuration();
 
+	if (heap_initialized)
+		return 0;
+
 	if (internal_conf->match_allocations)
 		RTE_LOG(DEBUG, EAL, "Hugepages will be freed exactly as allocated.\n");
 
@@ -1449,6 +1454,8 @@ int rte_eal_malloc_heap_populate(void)
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
 
+	heap_initialized = true;
+
 	/* add all IOVA-contiguous areas to the heap */
 	return rte_memseg_contig_walk(malloc_add_seg, NULL);
 }
-- 
2.25.1

[PATCH v3 6/7] eal_interrupts: don't reinitialize threads

[okaya]

From: Sinan Kaya <okaya@kernel.org>

Initialize interrupt thread once and keep a flag
for further init.

Signed-off-by: Sinan Kaya <okaya@kernel.org>
---
 lib/eal/linux/eal_interrupts.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/lib/eal/linux/eal_interrupts.c b/lib/eal/linux/eal_interrupts.c
index c9881143be..6a35b4aebd 100644
--- a/lib/eal/linux/eal_interrupts.c
+++ b/lib/eal/linux/eal_interrupts.c
@@ -91,6 +91,9 @@ static struct rte_intr_source_list intr_sources;
 /* interrupt handling thread */
 static pthread_t intr_thread;
 
+/* flag for initialization */
+static bool intr_initialized;
+
 /* VFIO interrupts */
 #ifdef VFIO_PRESENT
 
@@ -1175,6 +1178,9 @@ rte_eal_intr_init(void)
 {
 	int ret = 0;
 
+	if (intr_initialized)
+		return 0;
+
 	/* init the global interrupt source head */
 	TAILQ_INIT(&intr_sources);
 
@@ -1196,6 +1202,7 @@ rte_eal_intr_init(void)
 			"Failed to create thread for interrupt handling\n");
 	}
 
+	intr_initialized = true;
 	return ret;
 }
 
-- 
2.25.1

[PATCH v3 7/7] eal: initialize worker threads once

[okaya]

From: Sinan Kaya <okaya@kernel.org>

Initialize worker threads once and keep a flag
for other init calls.

Signed-off-by: Sinan Kaya <okaya@kernel.org>
---
 lib/eal/linux/eal.c | 60 ++++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/lib/eal/linux/eal.c b/lib/eal/linux/eal.c
index 650fb5f5a9..9a41bc66cf 100644
--- a/lib/eal/linux/eal.c
+++ b/lib/eal/linux/eal.c
@@ -80,6 +80,7 @@ int rte_cycles_vmware_tsc_map;
 
 
 static uint32_t run_once;
+static bool worker_initialized;
 
 int
 eal_clean_runtime_dir(void)
@@ -1254,42 +1255,45 @@ rte_eal_init(int argc, char **argv)
 		config->main_lcore, (uintptr_t)pthread_self(), cpuset,
 		ret == 0 ? "" : "...");
 
-	RTE_LCORE_FOREACH_WORKER(i) {
+	if (worker_initialized == false) {
+		RTE_LCORE_FOREACH_WORKER(i) {
 
 		/*
 		 * create communication pipes between main thread
 		 * and children
 		 */
-		if (pipe(lcore_config[i].pipe_main2worker) < 0)
-			rte_panic("Cannot create pipe\n");
-		if (pipe(lcore_config[i].pipe_worker2main) < 0)
-			rte_panic("Cannot create pipe\n");
-
-		lcore_config[i].state = WAIT;
-
-		/* create a thread for each lcore */
-		ret = eal_worker_thread_create(i);
-		if (ret != 0)
-			rte_panic("Cannot create thread\n");
-
-		/* Set thread_name for aid in debugging. */
-		snprintf(thread_name, sizeof(thread_name),
-			"rte-worker-%d", i);
-		rte_thread_set_name(lcore_config[i].thread_id, thread_name);
+			if (pipe(lcore_config[i].pipe_main2worker) < 0)
+				rte_panic("Cannot create pipe\n");
+			if (pipe(lcore_config[i].pipe_worker2main) < 0)
+				rte_panic("Cannot create pipe\n");
+
+			lcore_config[i].state = WAIT;
+
+			/* create a thread for each lcore */
+			ret = eal_worker_thread_create(i);
+			if (ret != 0)
+				rte_panic("Cannot create thread\n");
+
+			/* Set thread_name for aid in debugging. */
+			snprintf(thread_name, sizeof(thread_name),
+				"rte-worker-%d", i);
+			rte_thread_set_name(lcore_config[i].thread_id, thread_name);
+
+			ret = rte_thread_set_affinity_by_id(lcore_config[i].thread_id,
+				&lcore_config[i].cpuset);
+			if (ret != 0)
+				rte_panic("Cannot set affinity\n");
+		}
 
-		ret = rte_thread_set_affinity_by_id(lcore_config[i].thread_id,
-			&lcore_config[i].cpuset);
-		if (ret != 0)
-			rte_panic("Cannot set affinity\n");
+		/*
+		 * Launch a dummy function on all worker lcores, so that main lcore
+		 * knows they are all ready when this function returns.
+		 */
+		rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MAIN);
+		rte_eal_mp_wait_lcore();
+		worker_initialized = true;
 	}
 
-	/*
-	 * Launch a dummy function on all worker lcores, so that main lcore
-	 * knows they are all ready when this function returns.
-	 */
-	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MAIN);
-	rte_eal_mp_wait_lcore();
-
 	/* initialize services so vdevs register service during bus_probe. */
 	ret = rte_service_init();
 	if (ret) {
-- 
2.25.1