[RFC 1/3] vhost: add cleanup callback to FD entries

[Maxime Coquelin <maxime.coquelin@redhat.com>]

This patch adds an optional cleanup callback to
FD entries in the FD manager, which is called by the
FD event thread at FD entry removal time (when the read
or write callback has requested the FD removal).

Doing this, we avoid closing the file descriptor before it
is removed from the epoll FD set.

Fixes: 0e38b42bf61c ("vhost: manage FD with epoll")
Cc: stable@dpdk.org

Reported-by: David Marchand <david.marchand@redhat.com>
Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 lib/vhost/fd_man.c | 16 ++++++++++++----
 lib/vhost/fd_man.h |  3 ++-
 lib/vhost/socket.c |  4 ++--
 lib/vhost/vduse.c  |  6 +++---
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/lib/vhost/fd_man.c b/lib/vhost/fd_man.c
index 9bc7e50b93..c172ad8eff 100644
--- a/lib/vhost/fd_man.c
+++ b/lib/vhost/fd_man.c
@@ -25,6 +25,8 @@ struct fdentry {
 	int fd;		/* -1 indicates this entry is empty */
 	fd_cb rcb;	/* callback when this fd is readable. */
 	fd_cb wcb;	/* callback when this fd is writeable.*/
+	cleanup_cb ccb; /* callback when cleanup after entry
+			   removal from the fdset is needed */
 	void *dat;	/* fd context */
 	int busy;	/* whether this entry is being used in cb. */
 	LIST_ENTRY(fdentry) next;
@@ -150,7 +152,7 @@ fdset_init(const char *name)
 }
 
 static int
-fdset_insert_entry(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat)
+fdset_insert_entry(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, cleanup_cb ccb, void *dat)
 {
 	struct fdentry *pfdentry;
 
@@ -161,6 +163,7 @@ fdset_insert_entry(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat
 	pfdentry->fd  = fd;
 	pfdentry->rcb = rcb;
 	pfdentry->wcb = wcb;
+	pfdentry->ccb = ccb;
 	pfdentry->dat = dat;
 
 	LIST_INSERT_HEAD(&pfdset->fdlist, pfdentry, next);
@@ -210,7 +213,7 @@ fdset_find_entry_locked(struct fdset *pfdset, int fd)
  * Register the fd in the fdset with read/write handler and context.
  */
 int
-fdset_add(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat)
+fdset_add(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, cleanup_cb ccb, void *dat)
 {
 	struct epoll_event ev;
 	struct fdentry *pfdentry;
@@ -222,7 +225,7 @@ fdset_add(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat)
 	}
 
 	pthread_mutex_lock(&pfdset->fd_mutex);
-	ret = fdset_insert_entry(pfdset, fd, rcb, wcb, dat);
+	ret = fdset_insert_entry(pfdset, fd, rcb, wcb, ccb, dat);
 	if (ret < 0) {
 		VHOST_FDMAN_LOG(ERR, "failed to insert fdset entry");
 		pthread_mutex_unlock(&pfdset->fd_mutex);
@@ -331,6 +334,7 @@ fdset_event_dispatch(void *arg)
 {
 	int i;
 	fd_cb rcb, wcb;
+	cleanup_cb ccb;
 	void *dat;
 	int fd, numfds;
 	int remove1, remove2;
@@ -361,6 +365,7 @@ fdset_event_dispatch(void *arg)
 
 			rcb = pfdentry->rcb;
 			wcb = pfdentry->wcb;
+			ccb = pfdentry->ccb;
 			dat = pfdentry->dat;
 			pfdentry->busy = 1;
 
@@ -381,8 +386,11 @@ fdset_event_dispatch(void *arg)
 			 * fdentry not to be busy, so we can't call
 			 * fdset_del_locked().
 			 */
-			if (remove1 || remove2)
+			if (remove1 || remove2) {
 				fdset_del(pfdset, fd);
+				if (ccb)
+					ccb(fd, dat);
+			}
 		}
 
 		if (pfdset->destroy)
diff --git a/lib/vhost/fd_man.h b/lib/vhost/fd_man.h
index 6398343a6a..452c89135d 100644
--- a/lib/vhost/fd_man.h
+++ b/lib/vhost/fd_man.h
@@ -13,11 +13,12 @@ struct fdset;
 #define MAX_FDS 1024
 
 typedef void (*fd_cb)(int fd, void *dat, int *remove);
+typedef void (*cleanup_cb)(int fd, void *dat);
 
 struct fdset *fdset_init(const char *name);
 
 int fdset_add(struct fdset *pfdset, int fd,
-	fd_cb rcb, fd_cb wcb, void *dat);
+	fd_cb rcb, fd_cb wcb, cleanup_cb ccb, void *dat);
 
 void fdset_del(struct fdset *pfdset, int fd);
 int fdset_try_del(struct fdset *pfdset, int fd);
diff --git a/lib/vhost/socket.c b/lib/vhost/socket.c
index d29d15494c..e301efb367 100644
--- a/lib/vhost/socket.c
+++ b/lib/vhost/socket.c
@@ -263,7 +263,7 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
 	conn->vsocket = vsocket;
 	conn->vid = vid;
 	ret = fdset_add(vhost_user.fdset, fd, vhost_user_read_cb,
-			NULL, conn);
+			NULL, NULL, conn);
 	if (ret < 0) {
 		VHOST_CONFIG_LOG(vsocket->path, ERR,
 			"failed to add fd %d into vhost server fdset",
@@ -396,7 +396,7 @@ vhost_user_start_server(struct vhost_user_socket *vsocket)
 		goto err;
 
 	ret = fdset_add(vhost_user.fdset, fd, vhost_user_server_new_connection,
-		  NULL, vsocket);
+		  NULL, NULL, vsocket);
 	if (ret < 0) {
 		VHOST_CONFIG_LOG(path, ERR, "failed to add listen fd %d to vhost server fdset",
 			fd);
diff --git a/lib/vhost/vduse.c b/lib/vhost/vduse.c
index 8ba58555f9..9c10fbc684 100644
--- a/lib/vhost/vduse.c
+++ b/lib/vhost/vduse.c
@@ -218,7 +218,7 @@ vduse_vring_setup(struct virtio_net *dev, unsigned int index, bool reconnect)
 	}
 
 	if (vq == dev->cvq) {
-		ret = fdset_add(vduse.fdset, vq->kickfd, vduse_control_queue_event, NULL, dev);
+		ret = fdset_add(vduse.fdset, vq->kickfd, vduse_control_queue_event, NULL, NULL, dev);
 		if (ret) {
 			VHOST_CONFIG_LOG(dev->ifname, ERR,
 					"Failed to setup kickfd handler for VQ %u: %s",
@@ -590,7 +590,7 @@ vduse_reconnect_start_device(struct virtio_net *dev)
 		goto out_err;
 	}
 
-	ret = fdset_add(vduse.fdset, fd, vduse_reconnect_handler, NULL, dev);
+	ret = fdset_add(vduse.fdset, fd, vduse_reconnect_handler, NULL, NULL, dev);
 	if (ret) {
 		VHOST_CONFIG_LOG(dev->ifname, ERR, "Failed to add reconnect efd %d to vduse fdset",
 				fd);
@@ -787,7 +787,7 @@ vduse_device_create(const char *path, bool compliant_ol_flags)
 
 	dev->cvq = dev->virtqueue[max_queue_pairs * 2];
 
-	ret = fdset_add(vduse.fdset, dev->vduse_dev_fd, vduse_events_handler, NULL, dev);
+	ret = fdset_add(vduse.fdset, dev->vduse_dev_fd, vduse_events_handler, NULL, NULL, dev);
 	if (ret) {
 		VHOST_CONFIG_LOG(name, ERR, "Failed to add fd %d to vduse fdset",
 				dev->vduse_dev_fd);
-- 
2.47.1