From: Anatoly Burakov <anatoly.burakov@intel.com>
To: dev@dpdk.org
Cc: Bruce Richardson <bruce.richardson@intel.com>,
jianfeng.tan@intel.com, keith.wiles@intel.com
Subject: [dpdk-dev] [PATCH v4 2/5] eal: use file to check if secondary process is ready
Date: Fri, 2 Mar 2018 15:14:09 +0000 [thread overview]
Message-ID: <06149b2941712f7abe5c02602d444b5a6937ea2f.1520000413.git.anatoly.burakov@intel.com> (raw)
In-Reply-To: <a9cae982af2307c7fa59406723bca84004de5ff6.1520000413.git.anatoly.burakov@intel.com>
In-Reply-To: <df64cdb071cef1002de60c963701a49e1bc3f155.1519322682.git.anatoly.burakov@intel.com>
Previously, IPC would remove sockets it considers to be "inactive"
based on whether they have responded. We also need to prevent
sending messages to processes that are active, but haven't yet
finished initialization.
This will create a "init file" per socket which will be removed
after initialization is complete, to prevent primary process from
sending messages to a process that hasn't finished its
initialization.
Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
---
Notes:
v4: changed from "don't process messages until init
complete" to "don't send messages to processes
which haven't finished initializing", as the
former would have resulted in timeouts if init
took too long to complete.
v4: rework to not send any messages to secondary processes that
haven't yet initialized, so no need for message queue
v3: no changes
v2: no changes
lib/librte_eal/bsdapp/eal/eal.c | 7 ++
lib/librte_eal/common/eal_common_proc.c | 152 +++++++++++++++++++++++++++-----
lib/librte_eal/common/eal_private.h | 10 ++-
lib/librte_eal/linuxapp/eal/eal.c | 7 ++
4 files changed, 151 insertions(+), 25 deletions(-)
diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
index 4eafcb5..c003c16 100644
--- a/lib/librte_eal/bsdapp/eal/eal.c
+++ b/lib/librte_eal/bsdapp/eal/eal.c
@@ -694,6 +694,13 @@ rte_eal_init(int argc, char **argv)
return -1;
}
+ ret = rte_mp_channel_set_ready();
+ if (ret < 0) {
+ rte_eal_init_alert("Cannot finalize mp channel init\n");
+ rte_errno = EFAULT;
+ return -1;
+ }
+
rte_eal_mcfg_complete();
return fctret;
diff --git a/lib/librte_eal/common/eal_common_proc.c b/lib/librte_eal/common/eal_common_proc.c
index da7930f..4d227fe 100644
--- a/lib/librte_eal/common/eal_common_proc.c
+++ b/lib/librte_eal/common/eal_common_proc.c
@@ -13,6 +13,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/file.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>
@@ -33,6 +34,7 @@
static int mp_fd = -1;
static char mp_filter[PATH_MAX]; /* Filter for secondary process sockets */
static char mp_dir_path[PATH_MAX]; /* The directory path for all mp sockets */
+static char process_peer_name[PATH_MAX] = {0}; /* this process's peer name */
static pthread_mutex_t mp_mutex_action = PTHREAD_MUTEX_INITIALIZER;
struct action_entry {
@@ -91,6 +93,48 @@ find_sync_request(const char *dst, const char *act_name)
return r;
}
+static void
+create_socket_path(const char *name, char *buf, int len)
+{
+ const char *prefix = eal_mp_socket_path();
+
+ if (strlen(name) > 0)
+ snprintf(buf, len, "%s_%s", prefix, name);
+ else
+ snprintf(buf, len, "%s", prefix);
+}
+
+static void
+create_initfile_path(const char *name, char *buf, int len)
+{
+ const char *prefix = eal_mp_socket_path();
+
+ if (strlen(name) > 1)
+ snprintf(buf, len, "%sinit_%s", prefix, name);
+ else
+ snprintf(buf, len, "%sinit", prefix);
+}
+
+static const char *
+get_peer_name(const char *socket_full_path)
+{
+ char buf[PATH_MAX] = {0};
+ int len;
+
+ /* primary process has no peer name */
+ if (strcmp(socket_full_path, eal_mp_socket_path()) == 0)
+ return NULL;
+
+ /* construct dummy socket file name - make it one character long so that
+ * we hit the code path where underscores are added
+ */
+ create_socket_path("a", buf, sizeof(buf));
+
+ /* we want to get everything after /path/.rte_unix_, so discard 'a' */
+ len = strlen(buf) - 1;
+ return &socket_full_path[len];
+}
+
int
rte_eal_primary_proc_alive(const char *config_file_path)
{
@@ -290,8 +334,23 @@ mp_handle(void *arg __rte_unused)
static int
open_socket_fd(void)
{
+ char initfile[PATH_MAX] = {0};
struct sockaddr_un un;
- const char *prefix = eal_mp_socket_path();
+ int init_fd;
+
+ if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+ snprintf(process_peer_name, sizeof(process_peer_name),
+ "%d_%"PRIx64, getpid(), rte_rdtsc());
+
+ /* try to create initfile */
+ create_initfile_path(process_peer_name, initfile, sizeof(initfile));
+
+ init_fd = open(initfile, O_CREAT | O_RDWR);
+ if (init_fd < 0) {
+ RTE_LOG(ERR, EAL, "failed to open '%s': %s\n", initfile,
+ strerror(errno));
+ return -1;
+ }
mp_fd = socket(AF_UNIX, SOCK_DGRAM, 0);
if (mp_fd < 0) {
@@ -301,13 +360,11 @@ open_socket_fd(void)
memset(&un, 0, sizeof(un));
un.sun_family = AF_UNIX;
- if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- snprintf(un.sun_path, sizeof(un.sun_path), "%s", prefix);
- else {
- snprintf(un.sun_path, sizeof(un.sun_path), "%s_%d_%"PRIx64,
- prefix, getpid(), rte_rdtsc());
- }
+
+ create_socket_path(process_peer_name, un.sun_path, sizeof(un.sun_path));
+
unlink(un.sun_path); /* May still exist since last run */
+
if (bind(mp_fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
RTE_LOG(ERR, EAL, "failed to bind %s: %s\n",
un.sun_path, strerror(errno));
@@ -319,6 +376,31 @@ open_socket_fd(void)
return mp_fd;
}
+/* find corresponding init file */
+static int
+socket_is_ready(const char *peer_name)
+{
+ char initfile[PATH_MAX] = {0};
+ int fd;
+
+ /* construct lockfile filename */
+ create_initfile_path(peer_name, initfile, sizeof(initfile));
+
+ fd = open(initfile, O_RDWR);
+ if (fd >= 0) {
+ /* init file still exists, socket is not ready */
+ close(fd);
+ return 0;
+ }
+ if (fd < 0 && errno == ENOENT) {
+ /* init file not found, socket is ready */
+ return 1;
+ }
+ RTE_LOG(ERR, EAL, "Cannot open '%s': %s\n", initfile,
+ strerror(errno));
+ return -1;
+}
+
static int
unlink_sockets(const char *filter)
{
@@ -334,26 +416,30 @@ unlink_sockets(const char *filter)
dir_fd = dirfd(mp_dir);
while ((ent = readdir(mp_dir))) {
- if (fnmatch(filter, ent->d_name, 0) == 0)
+ if (fnmatch(filter, ent->d_name, 0) == 0) {
+ char path[PATH_MAX];
+
+ snprintf(path, sizeof(path), "%s/%s", mp_dir_path,
+ ent->d_name);
+
+ RTE_LOG(DEBUG, EAL, "Removing stale socket file '%s'\n",
+ ent->d_name);
unlinkat(dir_fd, ent->d_name, 0);
+ }
}
closedir(mp_dir);
return 0;
}
-static void
-unlink_socket_by_path(const char *path)
+int
+rte_mp_channel_set_ready(void)
{
- char *filename;
- char *fullpath = strdup(path);
+ char path[PATH_MAX] = {0};
- if (!fullpath)
- return;
- filename = basename(fullpath);
- unlink_sockets(filename);
- free(fullpath);
- RTE_LOG(INFO, EAL, "Remove socket %s\n", path);
+ create_initfile_path(process_peer_name, path, PATH_MAX);
+
+ return unlink(path);
}
int
@@ -444,10 +530,9 @@ send_msg(const char *dst_path, struct rte_mp_msg *msg, int type)
if (snd < 0) {
rte_errno = errno;
/* Check if it caused by peer process exits */
- if (errno == ECONNREFUSED) {
- /* We don't unlink the primary's socket here */
- if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- unlink_socket_by_path(dst_path);
+ if (errno == ECONNREFUSED &&
+ rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ unlink(dst_path);
return 0;
}
if (errno == ENOBUFS) {
@@ -490,14 +575,22 @@ mp_send(struct rte_mp_msg *msg, const char *peer, int type)
}
while ((ent = readdir(mp_dir))) {
char path[PATH_MAX];
+ const char *peer_name;
+ int ready;
if (fnmatch(mp_filter, ent->d_name, 0) != 0)
continue;
snprintf(path, sizeof(path), "%s/%s", mp_dir_path,
ent->d_name);
- if (send_msg(path, msg, type) < 0)
+ peer_name = get_peer_name(path);
+
+ /* only send if we can expect to receive a reply */
+ ready = socket_is_ready(peer_name);
+ if (ready < 0)
ret = -1;
+ else if (ready > 0)
+ ret = send_msg(path, msg, type);
}
closedir(mp_dir);
@@ -655,15 +748,26 @@ rte_mp_request(struct rte_mp_msg *req, struct rte_mp_reply *reply,
rte_errno = errno;
return -1;
}
-
while ((ent = readdir(mp_dir))) {
+ const char *peer_name;
char path[PATH_MAX];
+ int ready;
if (fnmatch(mp_filter, ent->d_name, 0) != 0)
continue;
snprintf(path, sizeof(path), "%s/%s", mp_dir_path,
ent->d_name);
+ peer_name = get_peer_name(path);
+
+ ready = socket_is_ready(peer_name);
+
+ if (ready < 0) {
+ ret = -1;
+ continue;
+ } else if (ready == 0) {
+ continue;
+ }
if (mp_request_one(path, req, reply, &end))
ret = -1;
diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h
index 0b28770..a3535a0 100644
--- a/lib/librte_eal/common/eal_private.h
+++ b/lib/librte_eal/common/eal_private.h
@@ -202,7 +202,15 @@ struct rte_bus *rte_bus_find_by_device_name(const char *str);
* 0 on success;
* (<0) on failure.
*/
-
int rte_mp_channel_init(void);
+/**
+ * Set unix channel for primary/secondary communication as ready.
+ *
+ * @return
+ * 0 on success;
+ * (<0) on failure.
+ */
+int rte_mp_channel_set_ready(void);
+
#endif /* _EAL_PRIVATE_H_ */
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 2ecd07b..a9af9c6 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -961,6 +961,13 @@ rte_eal_init(int argc, char **argv)
return -1;
}
+ ret = rte_mp_channel_set_ready();
+ if (ret < 0) {
+ rte_eal_init_alert("Cannot finalize mp channel init\n");
+ rte_errno = EFAULT;
+ return -1;
+ }
+
rte_eal_mcfg_complete();
return fctret;
--
2.7.4
next prev parent reply other threads:[~2018-03-02 15:14 UTC|newest]
Thread overview: 57+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-02-22 18:21 [dpdk-dev] [PATCH 1/3] eal: add internal flag indicating init has completed Anatoly Burakov
2018-02-22 18:21 ` [dpdk-dev] [PATCH 2/3] eal: don't process IPC messages before init finished Anatoly Burakov
2018-02-22 18:21 ` [dpdk-dev] [PATCH 3/3] eal: use locks to determine if secondary process is active Anatoly Burakov
2018-03-02 15:14 ` [dpdk-dev] [PATCH v4 1/5] eal: add internal flag indicating init has completed Anatoly Burakov
2018-03-07 16:56 ` [dpdk-dev] [PATCH v5 0/6] Improvements for DPDK IPC Anatoly Burakov
2018-03-13 17:42 ` [dpdk-dev] [PATCH v6 " Anatoly Burakov
2018-03-21 17:43 ` Thomas Monjalon
2018-03-13 17:42 ` [dpdk-dev] [PATCH v6 1/6] eal: add internal flag indicating init has completed Anatoly Burakov
2018-03-13 20:59 ` Bruce Richardson
2018-03-13 17:42 ` [dpdk-dev] [PATCH v6 2/6] eal: abstract away IPC socket path generation Anatoly Burakov
2018-03-13 17:42 ` [dpdk-dev] [PATCH v6 3/6] eal: don't hardcode socket filter value in IPC Anatoly Burakov
2018-03-13 17:42 ` [dpdk-dev] [PATCH v6 4/6] eal: lock IPC directory on init and send Anatoly Burakov
2018-03-13 17:42 ` [dpdk-dev] [PATCH v6 5/6] eal: simplify IPC sync request timeout code Anatoly Burakov
2018-03-13 17:42 ` [dpdk-dev] [PATCH v6 6/6] eal: ignore messages until init is complete Anatoly Burakov
2018-03-07 16:56 ` [dpdk-dev] [PATCH v5 1/6] eal: add internal flag indicating init has completed Anatoly Burakov
2018-03-07 16:56 ` [dpdk-dev] [PATCH v5 2/6] eal: abstract away IPC socket path generation Anatoly Burakov
2018-03-11 9:02 ` Tan, Jianfeng
2018-03-07 16:56 ` [dpdk-dev] [PATCH v5 3/6] eal: don't hardcode socket filter value in IPC Anatoly Burakov
2018-03-07 16:56 ` [dpdk-dev] [PATCH v5 4/6] eal: lock IPC directory on init and send Anatoly Burakov
2018-03-11 9:14 ` Tan, Jianfeng
2018-03-07 16:56 ` [dpdk-dev] [PATCH v5 5/6] eal: simplify IPC sync request timeout code Anatoly Burakov
2018-03-11 9:25 ` Tan, Jianfeng
2018-03-07 16:56 ` [dpdk-dev] [PATCH v5 6/6] eal: ignore messages until init is complete Anatoly Burakov
2018-03-12 1:42 ` Tan, Jianfeng
2018-03-12 8:58 ` Burakov, Anatoly
2018-03-02 15:14 ` Anatoly Burakov [this message]
2018-03-06 11:03 ` [dpdk-dev] [PATCH v4 2/5] eal: use file to check if secondary process is ready Burakov, Anatoly
2018-03-02 15:14 ` [dpdk-dev] [PATCH v4 3/5] eal: prevent secondary process init while sending messages Anatoly Burakov
2018-03-02 15:14 ` [dpdk-dev] [PATCH v4 4/5] eal: don't hardcode socket filter value in IPC Anatoly Burakov
2018-03-02 15:14 ` [dpdk-dev] [PATCH v4 5/5] eal: simplify IPC sync request timeout code Anatoly Burakov
2018-02-22 18:32 ` [dpdk-dev] [PATCH 1/3] eal: add internal flag indicating init has completed Burakov, Anatoly
2018-02-27 13:23 ` [dpdk-dev] [PATCH v2 1/5] " Anatoly Burakov
2018-02-27 14:35 ` [dpdk-dev] [PATCH v3 " Anatoly Burakov
2018-02-28 2:12 ` Tan, Jianfeng
2018-02-28 9:43 ` Burakov, Anatoly
2018-02-27 14:35 ` [dpdk-dev] [PATCH v3 2/5] eal: don't process IPC messages before init finished Anatoly Burakov
2018-02-28 1:09 ` Tan, Jianfeng
2018-02-28 9:45 ` Burakov, Anatoly
2018-02-28 4:00 ` Wiles, Keith
2018-02-28 9:47 ` Burakov, Anatoly
2018-02-27 14:35 ` [dpdk-dev] [PATCH v3 3/5] eal: use locks to determine if secondary process is active Anatoly Burakov
2018-02-28 1:26 ` Tan, Jianfeng
2018-02-28 10:15 ` Burakov, Anatoly
2018-02-28 4:17 ` Wiles, Keith
2018-02-28 10:17 ` Burakov, Anatoly
2018-02-27 14:35 ` [dpdk-dev] [PATCH v3 4/5] eal: prevent secondary process init while sending messages Anatoly Burakov
2018-02-28 1:58 ` Tan, Jianfeng
2018-02-28 10:19 ` Burakov, Anatoly
2018-02-28 15:49 ` Tan, Jianfeng
2018-02-27 14:35 ` [dpdk-dev] [PATCH v3 5/5] eal: don't hardcode socket filter value in IPC Anatoly Burakov
2018-02-28 1:52 ` Tan, Jianfeng
2018-02-28 10:21 ` Burakov, Anatoly
2018-02-28 15:01 ` Tan, Jianfeng
2018-02-27 13:23 ` [dpdk-dev] [PATCH v2 2/5] eal: don't process IPC messages before init finished Anatoly Burakov
2018-02-27 13:23 ` [dpdk-dev] [PATCH v2 3/5] eal: use locks to determine if secondary process is active Anatoly Burakov
2018-02-27 13:23 ` [dpdk-dev] [PATCH v2 4/5] eal: prevent secondary process init while sending messages Anatoly Burakov
2018-02-27 13:23 ` [dpdk-dev] [PATCH v2 5/5] eal: don't hardcode socket filter value in IPC Anatoly Burakov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=06149b2941712f7abe5c02602d444b5a6937ea2f.1520000413.git.anatoly.burakov@intel.com \
--to=anatoly.burakov@intel.com \
--cc=bruce.richardson@intel.com \
--cc=dev@dpdk.org \
--cc=jianfeng.tan@intel.com \
--cc=keith.wiles@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).