DPDK patches and discussions
 help / color / mirror / Atom feed
From: Yuanhan Liu <yuanhan.liu@linux.intel.com>
To: dev@dpdk.org
Cc: huawei.xie@intel.com, Traynor Kevin <kevin.traynor@intel.com>,
	marcandre.lureau@redhat.com,
	Yuanhan Liu <yuanhan.liu@linux.intel.com>
Subject: [dpdk-dev] [PATCH v3 3/6] vhost: add reconnect ability
Date: Tue,  7 Jun 2016 12:05:05 +0800	[thread overview]
Message-ID: <1465272308-3572-4-git-send-email-yuanhan.liu@linux.intel.com> (raw)
In-Reply-To: <1465272308-3572-1-git-send-email-yuanhan.liu@linux.intel.com>

Allow reconnecting on failure by default when:

- DPDK app starts first and QEMU (as the server) is not started yet.
  Without reconnecting, DPDK app would simply fail on vhost-user
  registration.

- QEMU restarts, say due to OS reboot.
  Without reconnecting, you can't re-establish the connection without
  restarting DPDK app.

This patch make it work well for both above cases. It simply creates
a new thread, and keep trying calling "connect()", until it succeeds.

The reconnect could be disabled when RTE_VHOST_USER_NO_RECONNECT flag
is set.

Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
---

v2: - create one thread only to handle all reconnects
    - update release note on reconnect

v3: - enable reconnect by default, which is a handy ability to have.
---
 doc/guides/rel_notes/release_16_07.rst       |   7 ++
 lib/librte_vhost/rte_virtio_net.h            |   1 +
 lib/librte_vhost/vhost_user/vhost-net-user.c | 103 +++++++++++++++++++++++++--
 3 files changed, 107 insertions(+), 4 deletions(-)

diff --git a/doc/guides/rel_notes/release_16_07.rst b/doc/guides/rel_notes/release_16_07.rst
index b997a20..107ed83 100644
--- a/doc/guides/rel_notes/release_16_07.rst
+++ b/doc/guides/rel_notes/release_16_07.rst
@@ -61,6 +61,13 @@ New Features
   the time of writing this, patches for QEMU are not merged yet, but it's
   likely to get merged in v2.7.
 
+  DPDK vhost-user will also try to reconnect by default when
+
+  * the first connect fails (when QEMU is not started yet)
+  * the connection is broken (when QEMU restarts)
+
+  It can be turned off if flag ``RTE_VHOST_USER_NO_RECONNECT`` is set.
+
 
 Resolved Issues
 ---------------
diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h
index 5f69e78..9caa622 100644
--- a/lib/librte_vhost/rte_virtio_net.h
+++ b/lib/librte_vhost/rte_virtio_net.h
@@ -52,6 +52,7 @@
 #include <rte_ether.h>
 
 #define RTE_VHOST_USER_CLIENT		(1ULL << 0)
+#define RTE_VHOST_USER_NO_RECONNECT	(1ULL << 1)
 
 /* Enum for virtqueue management. */
 enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM};
diff --git a/lib/librte_vhost/vhost_user/vhost-net-user.c b/lib/librte_vhost/vhost_user/vhost-net-user.c
index a480f9f..94f1b92 100644
--- a/lib/librte_vhost/vhost_user/vhost-net-user.c
+++ b/lib/librte_vhost/vhost_user/vhost-net-user.c
@@ -41,6 +41,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
+#include <sys/queue.h>
 #include <errno.h>
 #include <pthread.h>
 
@@ -60,6 +61,7 @@ struct vhost_user_socket {
 	char *path;
 	int listenfd;
 	bool is_server;
+	bool reconnect;
 };
 
 struct vhost_user_connection {
@@ -79,6 +81,7 @@ struct vhost_user {
 
 static void vhost_user_server_new_connection(int fd, void *data, int *remove);
 static void vhost_user_msg_handler(int fd, void *dat, int *remove);
+static int vhost_user_create_client(struct vhost_user_socket *vsocket);
 
 static struct vhost_user vhost_user = {
 	.fdset = {
@@ -305,6 +308,8 @@ vhost_user_msg_handler(int connfd, void *dat, int *remove)
 	vid = conn->vid;
 	ret = read_vhost_message(connfd, &msg);
 	if (ret <= 0 || msg.request >= VHOST_USER_MAX) {
+		struct vhost_user_socket *vsocket = conn->vsocket;
+
 		if (ret < 0)
 			RTE_LOG(ERR, VHOST_CONFIG,
 				"vhost read message failed\n");
@@ -320,6 +325,9 @@ vhost_user_msg_handler(int connfd, void *dat, int *remove)
 		free(conn);
 		vhost_destroy_device(vid);
 
+		if (vsocket->reconnect)
+			vhost_user_create_client(vsocket);
+
 		return;
 	}
 
@@ -471,6 +479,73 @@ err:
 	return -1;
 }
 
+struct vhost_user_reconnect {
+	struct sockaddr_un un;
+	int fd;
+	struct vhost_user_socket *vsocket;
+
+	TAILQ_ENTRY(vhost_user_reconnect) next;
+};
+
+TAILQ_HEAD(vhost_user_reconnect_tailq_list, vhost_user_reconnect);
+struct vhost_user_reconnect_list {
+	struct vhost_user_reconnect_tailq_list head;
+	pthread_mutex_t mutex;
+};
+
+static struct vhost_user_reconnect_list reconn_list;
+static pthread_t reconn_tid;
+
+static void *
+vhost_user_client_reconnect(void *arg __rte_unused)
+{
+	struct vhost_user_reconnect *reconn, *next;
+
+	while (1) {
+		pthread_mutex_lock(&reconn_list.mutex);
+
+		/*
+		 * An equal implementation of TAILQ_FOREACH_SAFE,
+		 * which does not exist on all platforms.
+		 */
+		for (reconn = TAILQ_FIRST(&reconn_list.head);
+		     reconn != NULL; reconn = next) {
+			next = TAILQ_NEXT(reconn, next);
+
+			if (connect(reconn->fd, (struct sockaddr *)&reconn->un,
+				    sizeof(reconn->un)) < 0)
+				continue;
+
+			RTE_LOG(INFO, VHOST_CONFIG,
+				"%s: connected\n", reconn->vsocket->path);
+			vhost_user_add_connection(reconn->fd, reconn->vsocket);
+			TAILQ_REMOVE(&reconn_list.head, reconn, next);
+			free(reconn);
+		}
+
+		pthread_mutex_unlock(&reconn_list.mutex);
+		sleep(1);
+	}
+
+	return NULL;
+}
+
+static int
+vhost_user_reconnect_init(void)
+{
+	int ret;
+
+	pthread_mutex_init(&reconn_list.mutex, NULL);
+	TAILQ_INIT(&reconn_list.head);
+
+	ret = pthread_create(&reconn_tid, NULL,
+			     vhost_user_client_reconnect, NULL);
+	if (ret < 0)
+		RTE_LOG(ERR, VHOST_CONFIG, "failed to create reconnect thread");
+
+	return ret;
+}
+
 static int
 vhost_user_create_client(struct vhost_user_socket *vsocket)
 {
@@ -478,20 +553,35 @@ vhost_user_create_client(struct vhost_user_socket *vsocket)
 	int ret;
 	struct sockaddr_un un;
 	const char *path = vsocket->path;
+	struct vhost_user_reconnect *reconn;
 
 	fd = create_unix_socket(path, &un, vsocket->is_server);
 	if (fd < 0)
 		return -1;
 
 	ret = connect(fd, (struct sockaddr *)&un, sizeof(un));
-	if (ret < 0) {
-		RTE_LOG(ERR, VHOST_CONFIG, "failed to connect to %s: %s\n",
-			path, strerror(errno));
+	if (ret == 0) {
+		vhost_user_add_connection(fd, vsocket);
+		return 0;
+	}
+
+	RTE_LOG(ERR, VHOST_CONFIG,
+		"failed to connect to %s: %s\n",
+		path, strerror(errno));
+
+	if (!vsocket->reconnect) {
 		close(fd);
 		return -1;
 	}
 
-	vhost_user_add_connection(fd, vsocket);
+	RTE_LOG(ERR, VHOST_CONFIG, "%s: reconnecting...\n", path);
+	reconn = malloc(sizeof(*reconn));
+	reconn->un = un;
+	reconn->fd = fd;
+	reconn->vsocket = vsocket;
+	pthread_mutex_lock(&reconn_list.mutex);
+	TAILQ_INSERT_TAIL(&reconn_list.head, reconn, next);
+	pthread_mutex_unlock(&reconn_list.mutex);
 
 	return 0;
 }
@@ -525,6 +615,11 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 	vsocket->path = strdup(path);
 
 	if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
+		vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
+		if (vsocket->reconnect && reconn_tid == 0) {
+			if (vhost_user_reconnect_init() < 0)
+				goto out;
+		}
 		ret = vhost_user_create_client(vsocket);
 	} else {
 		vsocket->is_server = true;
-- 
1.9.0

  parent reply	other threads:[~2016-06-07  4:04 UTC|newest]

Thread overview: 50+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2016-05-07  6:40 [dpdk-dev] [PATCH 0/6] vhost: add vhost-user client mode and " Yuanhan Liu
2016-05-07  6:40 ` [dpdk-dev] [PATCH 1/6] vhost: rename structs for enabling client mode Yuanhan Liu
2016-05-07  6:40 ` [dpdk-dev] [PATCH 2/6] vhost: add vhost-user " Yuanhan Liu
2016-05-09 10:33   ` Victor Kaplansky
2016-05-09 20:33     ` Yuanhan Liu
2016-05-09 20:30       ` Michael S. Tsirkin
2016-05-07  6:40 ` [dpdk-dev] [PATCH 3/6] vhost: add reconnect ability Yuanhan Liu
2016-05-09 16:47   ` Xie, Huawei
2016-05-09 18:12     ` Yuanhan Liu
2016-05-10  7:24       ` Xie, Huawei
2016-05-10  7:54         ` Michael S. Tsirkin
2016-05-10  8:07           ` Xie, Huawei
2016-05-10  8:42             ` Michael S. Tsirkin
2016-05-10  9:00               ` Xie, Huawei
2016-05-10  9:17                 ` Michael S. Tsirkin
2016-05-10 17:17                   ` Loftus, Ciara
2016-05-11 21:46                     ` Michael S. Tsirkin
2016-05-07  6:40 ` [dpdk-dev] [PATCH 4/6] vhost: workaround stale vring base Yuanhan Liu
2016-05-09 10:45   ` Victor Kaplansky
2016-05-09 13:39     ` Xie, Huawei
2016-05-09 18:23       ` Yuanhan Liu
2016-05-09 12:19   ` Michael S. Tsirkin
2016-05-09 16:25   ` Xie, Huawei
2016-05-09 18:22     ` Yuanhan Liu
2016-06-13 20:47       ` Michael S. Tsirkin
2016-05-10  8:21   ` Xie, Huawei
2016-05-07  6:40 ` [dpdk-dev] [PATCH 5/6] examples/vhost: add client and reconnect option Yuanhan Liu
2016-05-09 10:47   ` Victor Kaplansky
2016-05-07  6:40 ` [dpdk-dev] [PATCH 6/6] vhost: add pmd " Yuanhan Liu
2016-05-09 10:54   ` Victor Kaplansky
2016-05-09 18:26     ` Yuanhan Liu
2016-05-10  3:23 ` [dpdk-dev] [PATCH 0/6] vhost: add vhost-user client mode and reconnect ability Xu, Qian Q
2016-05-10 17:41   ` Yuanhan Liu
2016-05-13  6:16 ` [dpdk-dev] [PATCH v2 " Yuanhan Liu
2016-05-13  6:16   ` [dpdk-dev] [PATCH v2 1/6] vhost: rename structs for enabling client mode Yuanhan Liu
2016-05-13  6:16   ` [dpdk-dev] [PATCH v2 2/6] vhost: add vhost-user " Yuanhan Liu
2016-05-13  6:16   ` [dpdk-dev] [PATCH v2 3/6] vhost: add reconnect ability Yuanhan Liu
2016-05-13  6:16   ` [dpdk-dev] [PATCH v2 4/6] vhost: workaround stale vring base Yuanhan Liu
2016-05-13  6:16   ` [dpdk-dev] [PATCH v2 5/6] examples/vhost: add client and reconnect option Yuanhan Liu
2016-05-13  6:16   ` [dpdk-dev] [PATCH v2 6/6] vhost: add pmd " Yuanhan Liu
2016-05-25 17:45     ` Rich Lane
2016-05-26  8:01       ` Yuanhan Liu
2016-06-07  4:05   ` [dpdk-dev] [PATCH v3 0/6] vhost: add vhost-user client mode and reconnect ability Yuanhan Liu
2016-06-07  4:05     ` [dpdk-dev] [PATCH v3 1/6] vhost: rename structs for enabling client mode Yuanhan Liu
2016-06-07  4:05     ` [dpdk-dev] [PATCH v3 2/6] vhost: add vhost-user " Yuanhan Liu
2016-06-07  4:05     ` Yuanhan Liu [this message]
2016-06-07  4:05     ` [dpdk-dev] [PATCH v3 4/6] vhost: workaround stale vring base Yuanhan Liu
2016-06-07  4:05     ` [dpdk-dev] [PATCH v3 5/6] examples/vhost: add client option Yuanhan Liu
2016-06-07  4:05     ` [dpdk-dev] [PATCH v3 6/6] vhost: add pmd " Yuanhan Liu
2016-06-14 12:00     ` [dpdk-dev] [PATCH v3 0/6] vhost: add vhost-user client mode and reconnect ability Yuanhan Liu

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1465272308-3572-4-git-send-email-yuanhan.liu@linux.intel.com \
    --to=yuanhan.liu@linux.intel.com \
    --cc=dev@dpdk.org \
    --cc=huawei.xie@intel.com \
    --cc=kevin.traynor@intel.com \
    --cc=marcandre.lureau@redhat.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).