DPDK patches and discussions
 help / color / mirror / Atom feed
From: zhichaox.zeng@intel.com
To: dev@dpdk.org
Cc: stable@dpdk.org, qiming.yang@intel.com,
	david.marchand@redhat.com, stephen@networkplumber.org,
	mb@smartsharesystems.com, Zhichao Zeng <zhichaox.zeng@intel.com>,
	Bruce Richardson <bruce.richardson@intel.com>,
	Harman Kalra <hkalra@marvell.com>
Subject: [PATCH v4] lib/eal: fix segfaults due to thread exit order
Date: Wed, 15 Jun 2022 14:01:54 +0800	[thread overview]
Message-ID: <20220615060154.6905-1-zhichaox.zeng@intel.com> (raw)
In-Reply-To: <20220530134738.488602-1-zhichaox.zeng@intel.com>

From: Zhichao Zeng <zhichaox.zeng@intel.com>

The eal-intr-thread is not closed before memory cleanup in the
process of exiting. There is a small probability that when the
eal-intr-thread is about to use some pointers, the memory were
just cleaned, which cause the segment fault error caught by ASan.

This patch close the eal-intr-thread before memory cleanup when
exiting to avoid segment fault. And add some atomic operations
to avoid executing rte_eal_cleanup in the child process spawned
by fork() in some test cases, e.g. debug_autotest of dpdk-test.

Cc: stable@dpdk.org

---
v2:
add the same API for FreeBSD
---
v3:
fix rte_eal_cleanup crash in debug_autotest
---
v4:
shorten the prompt message and optimize the commit log

Suggested-by: David Marchand <david.marchand@redhat.com>
Signed-off-by: Zhichao Zeng <zhichaox.zeng@intel.com>
---
 lib/eal/common/eal_private.h     |  7 +++++++
 lib/eal/freebsd/eal.c            | 21 ++++++++++++++++++++-
 lib/eal/freebsd/eal_interrupts.c | 12 ++++++++++++
 lib/eal/linux/eal.c              | 20 +++++++++++++++++++-
 lib/eal/linux/eal_interrupts.c   | 12 ++++++++++++
 5 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/lib/eal/common/eal_private.h b/lib/eal/common/eal_private.h
index 44d14241f0..7adf41b7d7 100644
--- a/lib/eal/common/eal_private.h
+++ b/lib/eal/common/eal_private.h
@@ -152,6 +152,13 @@ int rte_eal_tailqs_init(void);
  */
 int rte_eal_intr_init(void);
 
+/**
+ * Destroy interrupt handling thread.
+ *
+ * This function is private to EAL.
+ */
+void rte_eal_intr_destroy(void);
+
 /**
  * Close the default log stream
  *
diff --git a/lib/eal/freebsd/eal.c b/lib/eal/freebsd/eal.c
index a6b20960f2..4882f27abd 100644
--- a/lib/eal/freebsd/eal.c
+++ b/lib/eal/freebsd/eal.c
@@ -72,6 +72,8 @@ struct lcore_config lcore_config[RTE_MAX_LCORE];
 /* used by rte_rdtsc() */
 int rte_cycles_vmware_tsc_map;
 
+/* used to judge the running status of the eal */
+static uint32_t run_once;
 
 int
 eal_clean_runtime_dir(void)
@@ -574,12 +576,22 @@ static void rte_eal_init_alert(const char *msg)
 	RTE_LOG(ERR, EAL, "%s\n", msg);
 }
 
+static void warn_parent(void)
+{
+	RTE_LOG(WARNING, EAL, "DPDK won't work in the child process\n");
+}
+
+static void scratch_child(void)
+{
+	/* Scratch run_once so that a call to rte_eal_cleanup won't crash... */
+	__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+}
+
 /* Launch threads, called at application init(). */
 int
 rte_eal_init(int argc, char **argv)
 {
 	int i, fctret, ret;
-	static uint32_t run_once;
 	uint32_t has_run = 0;
 	char cpuset[RTE_CPU_AFFINITY_STR_LEN];
 	char thread_name[RTE_MAX_THREAD_NAME_LEN];
@@ -883,6 +895,8 @@ rte_eal_init(int argc, char **argv)
 
 	eal_mcfg_complete();
 
+	pthread_atfork(NULL, warn_parent, scratch_child);
+
 	return fctret;
 }
 
@@ -891,8 +905,13 @@ rte_eal_cleanup(void)
 {
 	struct internal_config *internal_conf =
 		eal_get_internal_configuration();
+
+	if (__atomic_load_n(&run_once, __ATOMIC_RELAXED) == 0)
+		return 0;
+
 	rte_service_finalize();
 	rte_mp_channel_cleanup();
+	rte_eal_intr_destroy();
 	/* after this point, any DPDK pointers will become dangling */
 	rte_eal_memory_detach();
 	rte_eal_alarm_cleanup();
diff --git a/lib/eal/freebsd/eal_interrupts.c b/lib/eal/freebsd/eal_interrupts.c
index 9f720bdc8f..cac3859b06 100644
--- a/lib/eal/freebsd/eal_interrupts.c
+++ b/lib/eal/freebsd/eal_interrupts.c
@@ -648,6 +648,18 @@ rte_eal_intr_init(void)
 	return ret;
 }
 
+void
+rte_eal_intr_destroy(void)
+{
+	/* cancel the host thread to wait/handle the interrupt */
+	pthread_cancel(intr_thread);
+	pthread_join(intr_thread, NULL);
+
+	/* close kqueue */
+	close(kq);
+	kq = -1;
+}
+
 int
 rte_intr_rx_ctl(struct rte_intr_handle *intr_handle,
 		int epfd, int op, unsigned int vec, void *data)
diff --git a/lib/eal/linux/eal.c b/lib/eal/linux/eal.c
index 1ef263434a..effebb33a6 100644
--- a/lib/eal/linux/eal.c
+++ b/lib/eal/linux/eal.c
@@ -76,6 +76,8 @@ struct lcore_config lcore_config[RTE_MAX_LCORE];
 /* used by rte_rdtsc() */
 int rte_cycles_vmware_tsc_map;
 
+/* used to judge the running status of the eal */
+static uint32_t run_once;
 
 int
 eal_clean_runtime_dir(void)
@@ -857,12 +859,22 @@ is_iommu_enabled(void)
 	return n > 2;
 }
 
+static void warn_parent(void)
+{
+	RTE_LOG(WARNING, EAL, "DPDK won't work in the child process\n");
+}
+
+static void scratch_child(void)
+{
+	/* Scratch run_once so that a call to rte_eal_cleanup won't crash... */
+	__atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
+}
+
 /* Launch threads, called at application init(). */
 int
 rte_eal_init(int argc, char **argv)
 {
 	int i, fctret, ret;
-	static uint32_t run_once;
 	uint32_t has_run = 0;
 	const char *p;
 	static char logid[PATH_MAX];
@@ -1228,6 +1240,8 @@ rte_eal_init(int argc, char **argv)
 
 	eal_mcfg_complete();
 
+	pthread_atfork(NULL, warn_parent, scratch_child);
+
 	return fctret;
 }
 
@@ -1257,6 +1271,9 @@ rte_eal_cleanup(void)
 	struct internal_config *internal_conf =
 		eal_get_internal_configuration();
 
+	if (__atomic_load_n(&run_once, __ATOMIC_RELAXED) == 0)
+		return 0;
+
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
 			internal_conf->hugepage_file.unlink_existing)
 		rte_memseg_walk(mark_freeable, NULL);
@@ -1266,6 +1283,7 @@ rte_eal_cleanup(void)
 	vfio_mp_sync_cleanup();
 #endif
 	rte_mp_channel_cleanup();
+	rte_eal_intr_destroy();
 	/* after this point, any DPDK pointers will become dangling */
 	rte_eal_memory_detach();
 	eal_mp_dev_hotplug_cleanup();
diff --git a/lib/eal/linux/eal_interrupts.c b/lib/eal/linux/eal_interrupts.c
index d52ec8eb4c..7e9853e8e7 100644
--- a/lib/eal/linux/eal_interrupts.c
+++ b/lib/eal/linux/eal_interrupts.c
@@ -1199,6 +1199,18 @@ rte_eal_intr_init(void)
 	return ret;
 }
 
+void
+rte_eal_intr_destroy(void)
+{
+	/* cancel the host thread to wait/handle the interrupt */
+	pthread_cancel(intr_thread);
+	pthread_join(intr_thread, NULL);
+
+	/* close the pipe used by epoll */
+	close(intr_pipe.writefd);
+	close(intr_pipe.readfd);
+}
+
 static void
 eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle)
 {
-- 
2.25.1


  parent reply	other threads:[~2022-06-15  6:02 UTC|newest]

Thread overview: 23+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-05-17 16:04 [DPDK] eal/linux: " zhichaox.zeng
2022-05-17  8:29 ` Morten Brørup
2022-05-17 15:25 ` [EXT] " Harman Kalra
2022-05-18 14:39 ` [DPDK v2] lib/eal: " zhichaox.zeng
2022-05-23 11:16 ` [PATCH " zhichaox.zeng
2022-05-23 12:10   ` David Marchand
2022-05-23 13:00     ` David Marchand
2022-05-30 13:47   ` [PATCH v3] " zhichaox.zeng
2022-05-30 16:26     ` Stephen Hemminger
2022-05-30 16:28     ` Stephen Hemminger
2022-06-02  8:21       ` Zeng, ZhichaoX
2022-06-07 10:14     ` Zeng, ZhichaoX
2022-06-15  6:01     ` zhichaox.zeng [this message]
2022-06-24  1:42       ` [PATCH v4] " Zeng, ZhichaoX
2022-06-24  7:50         ` David Marchand
2022-06-30 10:38         ` Zeng, ZhichaoX
2022-06-30 12:24       ` Bruce Richardson
2022-09-06  2:51       ` [PATCH v5] lib/eal: fix segfaults in exiting Zhichao Zeng
2022-09-06 15:03         ` Stephen Hemminger
2022-09-07  8:53           ` Zeng, ZhichaoX
2022-10-11  5:25         ` [PATCH v6] " Zhichao Zeng
2022-10-11 14:04           ` Stephen Hemminger
2022-10-19  1:51             ` Zeng, ZhichaoX

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20220615060154.6905-1-zhichaox.zeng@intel.com \
    --to=zhichaox.zeng@intel.com \
    --cc=bruce.richardson@intel.com \
    --cc=david.marchand@redhat.com \
    --cc=dev@dpdk.org \
    --cc=hkalra@marvell.com \
    --cc=mb@smartsharesystems.com \
    --cc=qiming.yang@intel.com \
    --cc=stable@dpdk.org \
    --cc=stephen@networkplumber.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).