From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga18.intel.com (mga18.intel.com [134.134.136.126]) by dpdk.org (Postfix) with ESMTP id D4A904C8C for ; Wed, 28 Feb 2018 02:27:32 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga008.jf.intel.com ([10.7.209.65]) by orsmga106.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 27 Feb 2018 17:27:31 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.47,403,1515484800"; d="scan'208";a="21308785" Received: from fmsmsx104.amr.corp.intel.com ([10.18.124.202]) by orsmga008.jf.intel.com with ESMTP; 27 Feb 2018 17:27:31 -0800 Received: from fmsmsx122.amr.corp.intel.com (10.18.125.37) by fmsmsx104.amr.corp.intel.com (10.18.124.202) with Microsoft SMTP Server (TLS) id 14.3.319.2; Tue, 27 Feb 2018 17:27:15 -0800 Received: from shsmsx151.ccr.corp.intel.com (10.239.6.50) by fmsmsx122.amr.corp.intel.com (10.18.125.37) with Microsoft SMTP Server (TLS) id 14.3.319.2; Tue, 27 Feb 2018 17:27:15 -0800 Received: from shsmsx103.ccr.corp.intel.com ([169.254.4.116]) by SHSMSX151.ccr.corp.intel.com ([169.254.3.116]) with mapi id 14.03.0319.002; Wed, 28 Feb 2018 09:26:13 +0800 From: "Tan, Jianfeng" To: "Burakov, Anatoly" , "dev@dpdk.org" Thread-Topic: [PATCH v3 3/5] eal: use locks to determine if secondary process is active Thread-Index: AQHTr9hBR6WS72EXakuObUuc04v0iqO5AjNA Date: Wed, 28 Feb 2018 01:26:12 +0000 Message-ID: References: <31f6d9ef676fb1eb0a664c06d62d66f32876dcb6.1519672713.git.anatoly.burakov@intel.com> In-Reply-To: Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Subject: Re: [dpdk-dev] [PATCH v3 3/5] eal: use locks to determine if secondary process is active X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 28 Feb 2018 01:27:33 -0000 > -----Original Message----- > From: Burakov, Anatoly > Sent: Tuesday, February 27, 2018 10:36 PM > To: dev@dpdk.org > Cc: Tan, Jianfeng > Subject: [PATCH v3 3/5] eal: use locks to determine if secondary process = is > active >=20 > Previously, IPC would remove sockets it considers to be "inactive" > based on whether they have responded. To be more precise, it was not depending on if the other side responses or = not; it was depending on sendmsg return error, ECONNREFUSED. > Change this to create lock > files in addition to socket files, so that we can determine if > secondary process is active before attempting to communicate with > it. That way, we can distinguish secondaries that are alive but > are not responding, from those that have already died. I think, by the old way, we can also "distinguish secondaries that are aliv= e but are not responding, from those that have already died", can't we? Thanks, Jianfeng >=20 > Signed-off-by: Anatoly Burakov > --- >=20 > Notes: > v3: no changes >=20 > v2: no changes >=20 > lib/librte_eal/common/eal_common_proc.c | 204 > +++++++++++++++++++++++++++----- > 1 file changed, 175 insertions(+), 29 deletions(-) >=20 > diff --git a/lib/librte_eal/common/eal_common_proc.c > b/lib/librte_eal/common/eal_common_proc.c > index a6e24e6..7c87971 100644 > --- a/lib/librte_eal/common/eal_common_proc.c > +++ b/lib/librte_eal/common/eal_common_proc.c > @@ -13,6 +13,7 @@ > #include > #include > #include > +#include > #include > #include > #include > @@ -32,6 +33,7 @@ > #include "eal_internal_cfg.h" >=20 > static int mp_fd =3D -1; > +static int lock_fd =3D -1; > static char mp_filter[PATH_MAX]; /* Filter for secondary process socke= ts */ > static char mp_dir_path[PATH_MAX]; /* The directory path for all mp > sockets */ > static pthread_mutex_t mp_mutex_action =3D PTHREAD_MUTEX_INITIALIZER; > @@ -104,6 +106,46 @@ find_sync_request(const char *dst, const char > *act_name) > return r; > } >=20 > +static void > +create_socket_path(const char *name, char *buf, int len) > +{ > + const char *prefix =3D eal_mp_socket_path(); > + if (strlen(name) > 0) > + snprintf(buf, len, "%s_%s", prefix, name); > + else > + snprintf(buf, len, "%s", prefix); > +} > + > +static void > +create_lockfile_path(const char *name, char *buf, int len) > +{ > + const char *prefix =3D eal_mp_socket_path(); > + if (strlen(name) > 1) > + snprintf(buf, len, "%slock_%s", prefix, name); > + else > + snprintf(buf, len, "%slock", prefix); > +} > + > +static const char * > +get_peer_name(const char *socket_full_path) > +{ > + char buf[PATH_MAX] =3D {0}; > + int len; > + > + /* primary process has no peer name */ > + if (strcmp(socket_full_path, eal_mp_socket_path()) =3D=3D 0) > + return NULL; > + > + /* construct dummy socket file name - make it one character long so > that > + * we hit the code path where underscores are added > + */ > + create_socket_path("a", buf, sizeof(buf)); > + > + /* we want to get everything after /path/.rte_unix_, so discard 'a' */ > + len =3D strlen(buf) - 1; > + return &socket_full_path[len]; > +} > + > int > rte_eal_primary_proc_alive(const char *config_file_path) > { > @@ -332,8 +374,29 @@ mp_handle(void *arg __rte_unused) > static int > open_socket_fd(void) > { > + char peer_name[PATH_MAX] =3D {0}; > + char lockfile[PATH_MAX] =3D {0}; > struct sockaddr_un un; > - const char *prefix =3D eal_mp_socket_path(); > + > + if (rte_eal_process_type() =3D=3D RTE_PROC_SECONDARY) > + snprintf(peer_name, sizeof(peer_name), "%d_%"PRIx64, > + getpid(), rte_rdtsc()); > + > + /* try to create lockfile */ > + create_lockfile_path(peer_name, lockfile, sizeof(lockfile)); > + > + lock_fd =3D open(lockfile, O_CREAT | O_RDWR); > + if (lock_fd < 0) { > + RTE_LOG(ERR, EAL, "failed to open '%s': %s\n", lockfile, > + strerror(errno)); > + return -1; > + } > + if (flock(lock_fd, LOCK_EX | LOCK_NB)) { > + RTE_LOG(ERR, EAL, "failed to lock '%s': %s\n", lockfile, > + strerror(errno)); > + return -1; > + } > + /* no need to downgrade to shared lock */ >=20 > mp_fd =3D socket(AF_UNIX, SOCK_DGRAM, 0); > if (mp_fd < 0) { > @@ -343,13 +406,11 @@ open_socket_fd(void) >=20 > memset(&un, 0, sizeof(un)); > un.sun_family =3D AF_UNIX; > - if (rte_eal_process_type() =3D=3D RTE_PROC_PRIMARY) > - snprintf(un.sun_path, sizeof(un.sun_path), "%s", prefix); > - else { > - snprintf(un.sun_path, sizeof(un.sun_path), > "%s_%d_%"PRIx64, > - prefix, getpid(), rte_rdtsc()); > - } > + > + create_socket_path(peer_name, un.sun_path, sizeof(un.sun_path)); > + > unlink(un.sun_path); /* May still exist since last run */ > + > if (bind(mp_fd, (struct sockaddr *)&un, sizeof(un)) < 0) { > RTE_LOG(ERR, EAL, "failed to bind %s: %s\n", > un.sun_path, strerror(errno)); > @@ -361,6 +422,44 @@ open_socket_fd(void) > return mp_fd; > } >=20 > +/* find corresponding lock file and try to lock it */ > +static int > +socket_is_active(const char *peer_name) > +{ > + char lockfile[PATH_MAX] =3D {0}; > + int fd, ret =3D -1; > + > + /* construct lockfile filename */ > + create_lockfile_path(peer_name, lockfile, sizeof(lockfile)); > + > + /* try to lock it */ > + fd =3D open(lockfile, O_CREAT | O_RDWR); > + if (fd < 0) { > + RTE_LOG(ERR, EAL, "Cannot open '%s': %s\n", lockfile, > + strerror(errno)); > + return -1; > + } > + ret =3D flock(fd, LOCK_EX | LOCK_NB); > + if (ret < 0) { > + if (errno =3D=3D EWOULDBLOCK) { > + /* file is locked */ > + ret =3D 1; > + } else { > + RTE_LOG(ERR, EAL, "Cannot lock '%s': %s\n", lockfile, > + strerror(errno)); > + ret =3D -1; > + } > + } else { > + ret =3D 0; > + /* unlink lockfile automatically */ > + unlink(lockfile); > + flock(fd, LOCK_UN); > + } > + close(fd); > + > + return ret; > +} > + > static int > unlink_sockets(const char *filter) > { > @@ -376,28 +475,33 @@ unlink_sockets(const char *filter) > dir_fd =3D dirfd(mp_dir); >=20 > while ((ent =3D readdir(mp_dir))) { > - if (fnmatch(filter, ent->d_name, 0) =3D=3D 0) > + if (fnmatch(filter, ent->d_name, 0) =3D=3D 0) { > + const char *peer_name; > + char path[PATH_MAX]; > + int ret; > + > + snprintf(path, sizeof(path), "%s/%s", mp_dir_path, > + ent->d_name); > + peer_name =3D get_peer_name(path); > + > + ret =3D socket_is_active(peer_name); > + if (ret < 0) { > + RTE_LOG(ERR, EAL, "Error getting socket > active status\n"); > + return -1; > + } else if (ret =3D=3D 1) { > + RTE_LOG(ERR, EAL, "Socket is active (old > secondary process still running?)\n"); > + return -1; > + } > + RTE_LOG(DEBUG, EAL, "Removing stale socket file > '%s'\n", > + ent->d_name); > unlinkat(dir_fd, ent->d_name, 0); > + } > } >=20 > closedir(mp_dir); > return 0; > } >=20 > -static void > -unlink_socket_by_path(const char *path) > -{ > - char *filename; > - char *fullpath =3D strdup(path); > - > - if (!fullpath) > - return; > - filename =3D basename(fullpath); > - unlink_sockets(filename); > - free(fullpath); > - RTE_LOG(INFO, EAL, "Remove socket %s\n", path); > -} > - > int > rte_mp_channel_init(void) > { > @@ -487,10 +591,25 @@ send_msg(const char *dst_path, struct > rte_mp_msg *msg, int type) > rte_errno =3D errno; > /* Check if it caused by peer process exits */ > if (errno =3D=3D ECONNREFUSED) { > - /* We don't unlink the primary's socket here */ > - if (rte_eal_process_type() =3D=3D RTE_PROC_PRIMARY) > - unlink_socket_by_path(dst_path); > - return 0; > + const char *peer_name =3D get_peer_name(dst_path); > + int active, ret =3D 0; > + > + active =3D rte_eal_process_type() =3D=3D > RTE_PROC_PRIMARY ? > + socket_is_active(peer_name) : > + rte_eal_primary_proc_alive(NULL); > + > + if (active > 0) { > + RTE_LOG(ERR, EAL, "Couldn't communicate > with active peer\n"); > + } else if (active < 0) { > + RTE_LOG(ERR, EAL, "Couldn't get peer > status\n"); > + ret =3D -1; > + } else if (rte_eal_process_type() =3D=3D > RTE_PROC_PRIMARY) { > + /* peer isn't active anymore, so unlink its > + * socket. > + */ > + unlink(dst_path); > + } > + return ret; > } > if (errno =3D=3D ENOBUFS) { > RTE_LOG(ERR, EAL, "Peer cannot receive > message %s\n", > @@ -508,7 +627,7 @@ send_msg(const char *dst_path, struct rte_mp_msg > *msg, int type) > static int > mp_send(struct rte_mp_msg *msg, const char *peer, int type) > { > - int ret =3D 0; > + int dir_fd, ret =3D 0; > DIR *mp_dir; > struct dirent *ent; >=20 > @@ -530,15 +649,28 @@ mp_send(struct rte_mp_msg *msg, const char > *peer, int type) > rte_errno =3D errno; > return -1; > } > + dir_fd =3D dirfd(mp_dir); > while ((ent =3D readdir(mp_dir))) { > char path[PATH_MAX]; > + const char *peer_name; > + int active; >=20 > if (fnmatch(mp_filter, ent->d_name, 0) !=3D 0) > continue; >=20 > snprintf(path, sizeof(path), "%s/%s", mp_dir_path, > ent->d_name); > - if (send_msg(path, msg, type) < 0) > + peer_name =3D get_peer_name(path); > + > + /* only send if we can expect to receive a reply, otherwise > + * remove the socket. > + */ > + active =3D socket_is_active(peer_name); > + if (active < 0) > + ret =3D -1; > + else if (active =3D=3D 0) > + unlinkat(dir_fd, ent->d_name, 0); > + else if (active > 0 && send_msg(path, msg, type) < 0) > ret =3D -1; > } >=20 > @@ -663,7 +795,7 @@ int __rte_experimental > rte_mp_request(struct rte_mp_msg *req, struct rte_mp_reply *reply, > const struct timespec *ts) > { > - int ret =3D 0; > + int dir_fd, ret =3D 0; > DIR *mp_dir; > struct dirent *ent; > struct timeval now; > @@ -698,15 +830,29 @@ rte_mp_request(struct rte_mp_msg *req, struct > rte_mp_reply *reply, > rte_errno =3D errno; > return -1; > } > + dir_fd =3D dirfd(mp_dir); >=20 > while ((ent =3D readdir(mp_dir))) { > + const char *peer_name; > char path[PATH_MAX]; > + int active; >=20 > if (fnmatch(mp_filter, ent->d_name, 0) !=3D 0) > continue; >=20 > snprintf(path, sizeof(path), "%s/%s", mp_dir_path, > ent->d_name); > + peer_name =3D get_peer_name(path); > + > + active =3D socket_is_active(peer_name); > + > + if (active < 0) { > + ret =3D -1; > + break; > + } else if (active =3D=3D 0) { > + unlinkat(dir_fd, ent->d_name, 0); > + continue; > + } >=20 > if (mp_request_one(path, req, reply, &end)) > ret =3D -1; > -- > 2.7.4