[dpdk-dev] [RFC 06/24] vhost: move vhost_user_connection to trans_af_unix.c

[Stefan Hajnoczi <stefanha@redhat.com>]

The AF_UNIX transport can accept multiple client connections on a server
socket.  Each connection instantiates a separate vhost-user device,
which is stored as a vhost_user_connection.  This behavior is specific
to AF_UNIX and other transports may not support N connections per
socket endpoint.

Move struct vhost_user_connection to trans_af_unix.c and
conn_list/conn_mutex into struct af_unix_socket.

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 lib/librte_vhost/vhost.h         | 19 ++-----------
 lib/librte_vhost/socket.c        | 36 ++----------------------
 lib/librte_vhost/trans_af_unix.c | 60 ++++++++++++++++++++++++++++++++++++----
 3 files changed, 59 insertions(+), 56 deletions(-)

diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 7cbef04ab..734a8721d 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -302,13 +302,10 @@ struct virtio_net {
 	int			slave_req_fd;
 } __rte_cache_aligned;
 
-/* The vhost_user, vhost_user_socket, vhost_user_connection, and reconnect
- * declarations are temporary measures for moving AF_UNIX code into
- * trans_af_unix.c.  They will be cleaned up as socket.c is untangled from
- * trans_af_unix.c.
+/* The vhost_user, vhost_user_socket, and reconnect declarations are temporary
+ * measures for moving AF_UNIX code into trans_af_unix.c.  They will be cleaned
+ * up as socket.c is untangled from trans_af_unix.c.
  */
-TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
-
 /*
  * Every time rte_vhost_driver_register() is invoked, an associated
  * vhost_user_socket struct will be created.
@@ -319,8 +316,6 @@ TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
  * struct.
  */
 struct vhost_user_socket {
-	struct vhost_user_connection_list conn_list;
-	pthread_mutex_t conn_mutex;
 	char *path;
 	bool is_server;
 	bool reconnect;
@@ -341,14 +336,6 @@ struct vhost_user_socket {
 	struct vhost_transport_ops const *trans_ops;
 };
 
-struct vhost_user_connection {
-	struct vhost_user_socket *vsocket;
-	int connfd;
-	int vid;
-
-	TAILQ_ENTRY(vhost_user_connection) next;
-};
-
 #define MAX_VHOST_SOCKET 1024
 struct vhost_user {
 	struct vhost_user_socket *vsockets[MAX_VHOST_SOCKET];
diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index f8a96ab5f..4fd86fd5b 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -153,13 +153,6 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 		free(vsocket);
 		goto out;
 	}
-	TAILQ_INIT(&vsocket->conn_list);
-	ret = pthread_mutex_init(&vsocket->conn_mutex, NULL);
-	if (ret) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"error: failed to init connection mutex\n");
-		goto out_free;
-	}
 	vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
 
 	/*
@@ -186,14 +179,14 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 		vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
 		if (vsocket->reconnect && reconn_tid == 0) {
 			if (vhost_user_reconnect_init() != 0)
-				goto out_mutex;
+				goto out_free;
 		}
 	} else {
 		vsocket->is_server = true;
 	}
 	ret = trans_ops->socket_init(vsocket, flags);
 	if (ret < 0) {
-		goto out_mutex;
+		goto out_free;
 	}
 
 	vhost_user.vsockets[vhost_user.vsocket_cnt++] = vsocket;
@@ -201,11 +194,6 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
 	pthread_mutex_unlock(&vhost_user.mutex);
 	return ret;
 
-out_mutex:
-	if (pthread_mutex_destroy(&vsocket->conn_mutex)) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"error: failed to destroy connection mutex\n");
-	}
 out_free:
 	free(vsocket->path);
 	free(vsocket);
@@ -223,7 +211,6 @@ rte_vhost_driver_unregister(const char *path)
 {
 	int i;
 	int count;
-	struct vhost_user_connection *conn, *next;
 
 	pthread_mutex_lock(&vhost_user.mutex);
 
@@ -232,25 +219,6 @@ rte_vhost_driver_unregister(const char *path)
 
 		if (!strcmp(vsocket->path, path)) {
 			vsocket->trans_ops->socket_cleanup(vsocket);
-
-			pthread_mutex_lock(&vsocket->conn_mutex);
-			for (conn = TAILQ_FIRST(&vsocket->conn_list);
-			     conn != NULL;
-			     conn = next) {
-				next = TAILQ_NEXT(conn, next);
-
-				fdset_del(&vhost_user.fdset, conn->connfd);
-				RTE_LOG(INFO, VHOST_CONFIG,
-					"free connfd = %d for device '%s'\n",
-					conn->connfd, path);
-				close(conn->connfd);
-				vhost_destroy_device(conn->vid);
-				TAILQ_REMOVE(&vsocket->conn_list, conn, next);
-				free(conn);
-			}
-			pthread_mutex_unlock(&vsocket->conn_mutex);
-
-			pthread_mutex_destroy(&vsocket->conn_mutex);
 			free(vsocket->path);
 			free(vsocket);
 
diff --git a/lib/librte_vhost/trans_af_unix.c b/lib/librte_vhost/trans_af_unix.c
index 6c22093a4..747fd9690 100644
--- a/lib/librte_vhost/trans_af_unix.c
+++ b/lib/librte_vhost/trans_af_unix.c
@@ -44,8 +44,20 @@
 
 #define MAX_VIRTIO_BACKLOG 128
 
+TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
+
+struct vhost_user_connection {
+	struct vhost_user_socket *vsocket;
+	int connfd;
+	int vid;
+
+	TAILQ_ENTRY(vhost_user_connection) next;
+};
+
 struct af_unix_socket {
 	struct vhost_user_socket socket; /* must be the first field! */
+	struct vhost_user_connection_list conn_list;
+	pthread_mutex_t conn_mutex;
 	int socket_fd;
 	struct sockaddr_un un;
 };
@@ -144,6 +156,8 @@ send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num)
 static void
 vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
 {
+	struct af_unix_socket *s =
+		container_of(vsocket, struct af_unix_socket, socket);
 	int vid;
 	size_t size;
 	struct vhost_user_connection *conn;
@@ -194,9 +208,9 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
 		goto err;
 	}
 
-	pthread_mutex_lock(&vsocket->conn_mutex);
-	TAILQ_INSERT_TAIL(&vsocket->conn_list, conn, next);
-	pthread_mutex_unlock(&vsocket->conn_mutex);
+	pthread_mutex_lock(&s->conn_mutex);
+	TAILQ_INSERT_TAIL(&s->conn_list, conn, next);
+	pthread_mutex_unlock(&s->conn_mutex);
 	return;
 
 err:
@@ -223,6 +237,8 @@ vhost_user_read_cb(int connfd, void *dat, int *remove)
 {
 	struct vhost_user_connection *conn = dat;
 	struct vhost_user_socket *vsocket = conn->vsocket;
+	struct af_unix_socket *s =
+		container_of(vsocket, struct af_unix_socket, socket);
 	int ret;
 
 	ret = vhost_user_msg_handler(conn->vid, connfd);
@@ -234,9 +250,9 @@ vhost_user_read_cb(int connfd, void *dat, int *remove)
 		if (vsocket->notify_ops->destroy_connection)
 			vsocket->notify_ops->destroy_connection(conn->vid);
 
-		pthread_mutex_lock(&vsocket->conn_mutex);
-		TAILQ_REMOVE(&vsocket->conn_list, conn, next);
-		pthread_mutex_unlock(&vsocket->conn_mutex);
+		pthread_mutex_lock(&s->conn_mutex);
+		TAILQ_REMOVE(&s->conn_list, conn, next);
+		pthread_mutex_unlock(&s->conn_mutex);
 
 		free(conn);
 
@@ -507,6 +523,18 @@ static int
 af_unix_socket_init(struct vhost_user_socket *vsocket,
 		    uint64_t flags __rte_unused)
 {
+	struct af_unix_socket *s =
+		container_of(vsocket, struct af_unix_socket, socket);
+	int ret;
+
+	TAILQ_INIT(&s->conn_list);
+	ret = pthread_mutex_init(&s->conn_mutex, NULL);
+	if (ret) {
+		RTE_LOG(ERR, VHOST_CONFIG,
+			"error: failed to init connection mutex\n");
+		return -1;
+	}
+
 	return create_unix_socket(vsocket);
 }
 
@@ -515,6 +543,7 @@ af_unix_socket_cleanup(struct vhost_user_socket *vsocket)
 {
 	struct af_unix_socket *s =
 		container_of(vsocket, struct af_unix_socket, socket);
+	struct vhost_user_connection *conn, *next;
 
 	if (vsocket->is_server) {
 		fdset_del(&vhost_user.fdset, s->socket_fd);
@@ -523,6 +552,25 @@ af_unix_socket_cleanup(struct vhost_user_socket *vsocket)
 	} else if (vsocket->reconnect) {
 		vhost_user_remove_reconnect(vsocket);
 	}
+
+	pthread_mutex_lock(&s->conn_mutex);
+	for (conn = TAILQ_FIRST(&s->conn_list);
+	     conn != NULL;
+	     conn = next) {
+		next = TAILQ_NEXT(conn, next);
+
+		fdset_del(&vhost_user.fdset, conn->connfd);
+		RTE_LOG(INFO, VHOST_CONFIG,
+			"free connfd = %d for device '%s'\n",
+			conn->connfd, vsocket->path);
+		close(conn->connfd);
+		vhost_destroy_device(conn->vid);
+		TAILQ_REMOVE(&s->conn_list, conn, next);
+		free(conn);
+	}
+	pthread_mutex_unlock(&s->conn_mutex);
+
+	pthread_mutex_destroy(&s->conn_mutex);
 }
 
 static int
-- 
2.14.3