From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id 05CFF5F1D for ; Fri, 2 Mar 2018 16:14:18 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga003.jf.intel.com ([10.7.209.27]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 02 Mar 2018 07:14:14 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,412,1515484800"; d="scan'208";a="31929515" Received: from irvmail001.ir.intel.com ([163.33.26.43]) by orsmga003.jf.intel.com with ESMTP; 02 Mar 2018 07:14:13 -0800 Received: from sivswdev01.ir.intel.com (sivswdev01.ir.intel.com [10.237.217.45]) by irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id w22FEC6f001228; Fri, 2 Mar 2018 15:14:12 GMT Received: from sivswdev01.ir.intel.com (localhost [127.0.0.1]) by sivswdev01.ir.intel.com with ESMTP id w22FECuM032124; Fri, 2 Mar 2018 15:14:12 GMT Received: (from aburakov@localhost) by sivswdev01.ir.intel.com with LOCAL id w22FECWi032119; Fri, 2 Mar 2018 15:14:12 GMT From: Anatoly Burakov To: dev@dpdk.org Cc: Bruce Richardson , jianfeng.tan@intel.com, keith.wiles@intel.com Date: Fri, 2 Mar 2018 15:14:09 +0000 Message-Id: <06149b2941712f7abe5c02602d444b5a6937ea2f.1520000413.git.anatoly.burakov@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: References: In-Reply-To: References: Subject: [dpdk-dev] [PATCH v4 2/5] eal: use file to check if secondary process is ready X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 02 Mar 2018 15:14:19 -0000 Previously, IPC would remove sockets it considers to be "inactive" based on whether they have responded. We also need to prevent sending messages to processes that are active, but haven't yet finished initialization. This will create a "init file" per socket which will be removed after initialization is complete, to prevent primary process from sending messages to a process that hasn't finished its initialization. Signed-off-by: Anatoly Burakov --- Notes: v4: changed from "don't process messages until init complete" to "don't send messages to processes which haven't finished initializing", as the former would have resulted in timeouts if init took too long to complete. v4: rework to not send any messages to secondary processes that haven't yet initialized, so no need for message queue v3: no changes v2: no changes lib/librte_eal/bsdapp/eal/eal.c | 7 ++ lib/librte_eal/common/eal_common_proc.c | 152 +++++++++++++++++++++++++++----- lib/librte_eal/common/eal_private.h | 10 ++- lib/librte_eal/linuxapp/eal/eal.c | 7 ++ 4 files changed, 151 insertions(+), 25 deletions(-) diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c index 4eafcb5..c003c16 100644 --- a/lib/librte_eal/bsdapp/eal/eal.c +++ b/lib/librte_eal/bsdapp/eal/eal.c @@ -694,6 +694,13 @@ rte_eal_init(int argc, char **argv) return -1; } + ret = rte_mp_channel_set_ready(); + if (ret < 0) { + rte_eal_init_alert("Cannot finalize mp channel init\n"); + rte_errno = EFAULT; + return -1; + } + rte_eal_mcfg_complete(); return fctret; diff --git a/lib/librte_eal/common/eal_common_proc.c b/lib/librte_eal/common/eal_common_proc.c index da7930f..4d227fe 100644 --- a/lib/librte_eal/common/eal_common_proc.c +++ b/lib/librte_eal/common/eal_common_proc.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ static int mp_fd = -1; static char mp_filter[PATH_MAX]; /* Filter for secondary process sockets */ static char mp_dir_path[PATH_MAX]; /* The directory path for all mp sockets */ +static char process_peer_name[PATH_MAX] = {0}; /* this process's peer name */ static pthread_mutex_t mp_mutex_action = PTHREAD_MUTEX_INITIALIZER; struct action_entry { @@ -91,6 +93,48 @@ find_sync_request(const char *dst, const char *act_name) return r; } +static void +create_socket_path(const char *name, char *buf, int len) +{ + const char *prefix = eal_mp_socket_path(); + + if (strlen(name) > 0) + snprintf(buf, len, "%s_%s", prefix, name); + else + snprintf(buf, len, "%s", prefix); +} + +static void +create_initfile_path(const char *name, char *buf, int len) +{ + const char *prefix = eal_mp_socket_path(); + + if (strlen(name) > 1) + snprintf(buf, len, "%sinit_%s", prefix, name); + else + snprintf(buf, len, "%sinit", prefix); +} + +static const char * +get_peer_name(const char *socket_full_path) +{ + char buf[PATH_MAX] = {0}; + int len; + + /* primary process has no peer name */ + if (strcmp(socket_full_path, eal_mp_socket_path()) == 0) + return NULL; + + /* construct dummy socket file name - make it one character long so that + * we hit the code path where underscores are added + */ + create_socket_path("a", buf, sizeof(buf)); + + /* we want to get everything after /path/.rte_unix_, so discard 'a' */ + len = strlen(buf) - 1; + return &socket_full_path[len]; +} + int rte_eal_primary_proc_alive(const char *config_file_path) { @@ -290,8 +334,23 @@ mp_handle(void *arg __rte_unused) static int open_socket_fd(void) { + char initfile[PATH_MAX] = {0}; struct sockaddr_un un; - const char *prefix = eal_mp_socket_path(); + int init_fd; + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) + snprintf(process_peer_name, sizeof(process_peer_name), + "%d_%"PRIx64, getpid(), rte_rdtsc()); + + /* try to create initfile */ + create_initfile_path(process_peer_name, initfile, sizeof(initfile)); + + init_fd = open(initfile, O_CREAT | O_RDWR); + if (init_fd < 0) { + RTE_LOG(ERR, EAL, "failed to open '%s': %s\n", initfile, + strerror(errno)); + return -1; + } mp_fd = socket(AF_UNIX, SOCK_DGRAM, 0); if (mp_fd < 0) { @@ -301,13 +360,11 @@ open_socket_fd(void) memset(&un, 0, sizeof(un)); un.sun_family = AF_UNIX; - if (rte_eal_process_type() == RTE_PROC_PRIMARY) - snprintf(un.sun_path, sizeof(un.sun_path), "%s", prefix); - else { - snprintf(un.sun_path, sizeof(un.sun_path), "%s_%d_%"PRIx64, - prefix, getpid(), rte_rdtsc()); - } + + create_socket_path(process_peer_name, un.sun_path, sizeof(un.sun_path)); + unlink(un.sun_path); /* May still exist since last run */ + if (bind(mp_fd, (struct sockaddr *)&un, sizeof(un)) < 0) { RTE_LOG(ERR, EAL, "failed to bind %s: %s\n", un.sun_path, strerror(errno)); @@ -319,6 +376,31 @@ open_socket_fd(void) return mp_fd; } +/* find corresponding init file */ +static int +socket_is_ready(const char *peer_name) +{ + char initfile[PATH_MAX] = {0}; + int fd; + + /* construct lockfile filename */ + create_initfile_path(peer_name, initfile, sizeof(initfile)); + + fd = open(initfile, O_RDWR); + if (fd >= 0) { + /* init file still exists, socket is not ready */ + close(fd); + return 0; + } + if (fd < 0 && errno == ENOENT) { + /* init file not found, socket is ready */ + return 1; + } + RTE_LOG(ERR, EAL, "Cannot open '%s': %s\n", initfile, + strerror(errno)); + return -1; +} + static int unlink_sockets(const char *filter) { @@ -334,26 +416,30 @@ unlink_sockets(const char *filter) dir_fd = dirfd(mp_dir); while ((ent = readdir(mp_dir))) { - if (fnmatch(filter, ent->d_name, 0) == 0) + if (fnmatch(filter, ent->d_name, 0) == 0) { + char path[PATH_MAX]; + + snprintf(path, sizeof(path), "%s/%s", mp_dir_path, + ent->d_name); + + RTE_LOG(DEBUG, EAL, "Removing stale socket file '%s'\n", + ent->d_name); unlinkat(dir_fd, ent->d_name, 0); + } } closedir(mp_dir); return 0; } -static void -unlink_socket_by_path(const char *path) +int +rte_mp_channel_set_ready(void) { - char *filename; - char *fullpath = strdup(path); + char path[PATH_MAX] = {0}; - if (!fullpath) - return; - filename = basename(fullpath); - unlink_sockets(filename); - free(fullpath); - RTE_LOG(INFO, EAL, "Remove socket %s\n", path); + create_initfile_path(process_peer_name, path, PATH_MAX); + + return unlink(path); } int @@ -444,10 +530,9 @@ send_msg(const char *dst_path, struct rte_mp_msg *msg, int type) if (snd < 0) { rte_errno = errno; /* Check if it caused by peer process exits */ - if (errno == ECONNREFUSED) { - /* We don't unlink the primary's socket here */ - if (rte_eal_process_type() == RTE_PROC_PRIMARY) - unlink_socket_by_path(dst_path); + if (errno == ECONNREFUSED && + rte_eal_process_type() == RTE_PROC_PRIMARY) { + unlink(dst_path); return 0; } if (errno == ENOBUFS) { @@ -490,14 +575,22 @@ mp_send(struct rte_mp_msg *msg, const char *peer, int type) } while ((ent = readdir(mp_dir))) { char path[PATH_MAX]; + const char *peer_name; + int ready; if (fnmatch(mp_filter, ent->d_name, 0) != 0) continue; snprintf(path, sizeof(path), "%s/%s", mp_dir_path, ent->d_name); - if (send_msg(path, msg, type) < 0) + peer_name = get_peer_name(path); + + /* only send if we can expect to receive a reply */ + ready = socket_is_ready(peer_name); + if (ready < 0) ret = -1; + else if (ready > 0) + ret = send_msg(path, msg, type); } closedir(mp_dir); @@ -655,15 +748,26 @@ rte_mp_request(struct rte_mp_msg *req, struct rte_mp_reply *reply, rte_errno = errno; return -1; } - while ((ent = readdir(mp_dir))) { + const char *peer_name; char path[PATH_MAX]; + int ready; if (fnmatch(mp_filter, ent->d_name, 0) != 0) continue; snprintf(path, sizeof(path), "%s/%s", mp_dir_path, ent->d_name); + peer_name = get_peer_name(path); + + ready = socket_is_ready(peer_name); + + if (ready < 0) { + ret = -1; + continue; + } else if (ready == 0) { + continue; + } if (mp_request_one(path, req, reply, &end)) ret = -1; diff --git a/lib/librte_eal/common/eal_private.h b/lib/librte_eal/common/eal_private.h index 0b28770..a3535a0 100644 --- a/lib/librte_eal/common/eal_private.h +++ b/lib/librte_eal/common/eal_private.h @@ -202,7 +202,15 @@ struct rte_bus *rte_bus_find_by_device_name(const char *str); * 0 on success; * (<0) on failure. */ - int rte_mp_channel_init(void); +/** + * Set unix channel for primary/secondary communication as ready. + * + * @return + * 0 on success; + * (<0) on failure. + */ +int rte_mp_channel_set_ready(void); + #endif /* _EAL_PRIVATE_H_ */ diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c index 2ecd07b..a9af9c6 100644 --- a/lib/librte_eal/linuxapp/eal/eal.c +++ b/lib/librte_eal/linuxapp/eal/eal.c @@ -961,6 +961,13 @@ rte_eal_init(int argc, char **argv) return -1; } + ret = rte_mp_channel_set_ready(); + if (ret < 0) { + rte_eal_init_alert("Cannot finalize mp channel init\n"); + rte_errno = EFAULT; + return -1; + } + rte_eal_mcfg_complete(); return fctret; -- 2.7.4