[dpdk-dev] [PATCH v8] eal_interrupts: add option for pending callback unregister

[dpdk-dev] [PATCH v8] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 .../common/include/rte_interrupts.h           | 32 +++++++
 lib/librte_eal/linuxapp/eal/eal_interrupts.c  | 85 ++++++++++++++++++-
 lib/librte_eal/rte_eal_version.map            |  1 +
 3 files changed, 116 insertions(+), 2 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..13d73a137 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * It unregisters the callback according to the specified interrupt handle,
+ * after it's no longer acive. Failes if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 3fe78260d..989e4d72a 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -318,6 +318,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

Re: [dpdk-dev] [PATCH v8] eal_interrupts: add option for pending callback unregister

[Thomas Monjalon]

Someone for a review please?

17/12/2018 13:30, Jakub Grajciar:
> use case: if callback is used to receive message form socket,
> and the message received is disconnect/error, this callback needs
> to be unregistered, but cannot because it is still active.
> 
> With this patch it is possible to mark the callback to be
> unregistered once the interrupt process is done with this
> interrupt source.
> 
> Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
> ---
>  .../common/include/rte_interrupts.h           | 32 +++++++
>  lib/librte_eal/linuxapp/eal/eal_interrupts.c  | 85 ++++++++++++++++++-
>  lib/librte_eal/rte_eal_version.map            |  1 +
>  3 files changed, 116 insertions(+), 2 deletions(-)
> 
> v2:
> - fix coding syle
> 
> v3:
> - fix locks
> 
> v4:
> - update version map
> 
> v6:
> - add _rte_experimental to rte_intr_callback_unregister_pending
> 
> v8:
> - fix compilation errors
> 
> diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
> index d751a6378..13d73a137 100644
> --- a/lib/librte_eal/common/include/rte_interrupts.h
> +++ b/lib/librte_eal/common/include/rte_interrupts.h
> @@ -6,6 +6,7 @@
>  #define _RTE_INTERRUPTS_H_
>  
>  #include <rte_common.h>
> +#include <rte_compat.h>
>  
>  /**
>   * @file
> @@ -24,6 +25,13 @@ struct rte_intr_handle;
>  /** Function to be registered for the specific interrupt */
>  typedef void (*rte_intr_callback_fn)(void *cb_arg);
>  
> +/**
> + * Function to call after a callback is unregistered.
> + * Can be used to close fd and free cb_arg.
> + */
> +typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
> +						void *cb_arg);
> +
>  #include "rte_eal_interrupts.h"
>  
>  /**
> @@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
>  int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
>  				rte_intr_callback_fn cb, void *cb_arg);
>  
> +/**
> + * It unregisters the callback according to the specified interrupt handle,
> + * after it's no longer acive. Failes if source is not active.
> + *
> + * @param intr_handle
> + *  pointer to the interrupt handle.
> + * @param cb
> + *  callback address.
> + * @param cb_arg
> + *  address of parameter for callback, (void *)-1 means to remove all
> + *  registered which has the same callback address.
> + * @param ucb_fn
> + *  callback to call before cb is unregistered (optional).
> + *  can be used to close fd and free cb_arg.
> + *
> + * @return
> + *  - On success, return the number of callback entities marked for remove.
> + *  - On failure, a negative value.
> + */
> +int __rte_experimental
> +rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
> +				rte_intr_callback_fn cb_fn, void *cb_arg,
> +				rte_intr_unregister_callback_fn ucb_fn);
> +
>  /**
>   * It enables the interrupt for the specified handle.
>   *
> diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
> index cbac451e1..be0dd4490 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
> @@ -76,6 +76,8 @@ struct rte_intr_callback {
>  	TAILQ_ENTRY(rte_intr_callback) next;
>  	rte_intr_callback_fn cb_fn;  /**< callback address */
>  	void *cb_arg;                /**< parameter for callback */
> +	uint8_t pending_delete;      /**< delete after callback is called */
> +	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
>  };
>  
>  struct rte_intr_source {
> @@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
>  	}
>  	callback->cb_fn = cb;
>  	callback->cb_arg = cb_arg;
> +	callback->pending_delete = 0;
> +	callback->ucb_fn = NULL;
>  
>  	rte_spinlock_lock(&intr_lock);
>  
> @@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
>  	return ret;
>  }
>  
> +int __rte_experimental
> +rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
> +				rte_intr_callback_fn cb_fn, void *cb_arg,
> +				rte_intr_unregister_callback_fn ucb_fn)
> +{
> +	int ret;
> +	struct rte_intr_source *src;
> +	struct rte_intr_callback *cb, *next;
> +
> +	/* do parameter checking first */
> +	if (intr_handle == NULL || intr_handle->fd < 0) {
> +		RTE_LOG(ERR, EAL,
> +		"Unregistering with invalid input parameter\n");
> +		return -EINVAL;
> +	}
> +
> +	rte_spinlock_lock(&intr_lock);
> +
> +	/* check if the insterrupt source for the fd is existent */
> +	TAILQ_FOREACH(src, &intr_sources, next)
> +		if (src->intr_handle.fd == intr_handle->fd)
> +			break;
> +
> +	/* No interrupt source registered for the fd */
> +	if (src == NULL) {
> +		ret = -ENOENT;
> +
> +	/* only usable if the source is active */
> +	} else if (src->active == 0) {
> +		ret = -EAGAIN;
> +
> +	} else {
> +		ret = 0;
> +
> +		/* walk through the callbacks and mark all that match. */
> +		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
> +			next = TAILQ_NEXT(cb, next);
> +			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
> +					cb->cb_arg == cb_arg)) {
> +				cb->pending_delete = 1;
> +				cb->ucb_fn = ucb_fn;
> +				ret++;
> +			}
> +		}
> +	}
> +
> +	rte_spinlock_unlock(&intr_lock);
> +
> +	return ret;
> +}
> +
>  int
>  rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
>  			rte_intr_callback_fn cb_fn, void *cb_arg)
> @@ -698,7 +753,7 @@ static int
>  eal_intr_process_interrupts(struct epoll_event *events, int nfds)
>  {
>  	bool call = false;
> -	int n, bytes_read;
> +	int n, bytes_read, rv;
>  	struct rte_intr_source *src;
>  	struct rte_intr_callback *cb, *next;
>  	union rte_intr_read_buffer buf;
> @@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
>  				rte_spinlock_lock(&intr_lock);
>  			}
>  		}
> -
>  		/* we done with that interrupt source, release it. */
>  		src->active = 0;
> +
> +		rv = 0;
> +
> +		/* check if any callback are supposed to be removed */
> +		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
> +			next = TAILQ_NEXT(cb, next);
> +			if (cb->pending_delete) {
> +				TAILQ_REMOVE(&src->callbacks, cb, next);
> +				if (cb->ucb_fn)
> +					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
> +				free(cb);
> +				rv++;
> +			}
> +		}
> +
> +		/* all callbacks for that source are removed. */
> +		if (TAILQ_EMPTY(&src->callbacks)) {
> +			TAILQ_REMOVE(&intr_sources, src, next);
> +			free(src);
> +		}
> +
> +		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
> +		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
> +			rte_spinlock_unlock(&intr_lock);
> +			return -EPIPE;
> +		}
> +
>  		rte_spinlock_unlock(&intr_lock);
>  	}
>  
> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index 3fe78260d..989e4d72a 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -318,6 +318,7 @@ EXPERIMENTAL {
>  	rte_fbarray_is_used;
>  	rte_fbarray_set_free;
>  	rte_fbarray_set_used;
> +	rte_intr_callback_unregister_pending;
>  	rte_log_register_type_and_pick_level;
>  	rte_malloc_dump_heaps;
>  	rte_malloc_heap_create;
> 

Re: [dpdk-dev] [PATCH v8] eal_interrupts: add option for pending callback unregister

[Thomas Monjalon]

Sorry for the delay in review.
I hoped someone else would review this patch.
Let's get this patch in DPDK 19.05, after doing some small changes (see below).

17/12/2018 13:30, Jakub Grajciar:
> use case: if callback is used to receive message form socket,
> and the message received is disconnect/error, this callback needs
> to be unregistered, but cannot because it is still active.
> 
> With this patch it is possible to mark the callback to be
> unregistered once the interrupt process is done with this
> interrupt source.
> 
> Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
> ---
>  .../common/include/rte_interrupts.h           | 32 +++++++
>  lib/librte_eal/linuxapp/eal/eal_interrupts.c  | 85 ++++++++++++++++++-
>  lib/librte_eal/rte_eal_version.map            |  1 +
>  3 files changed, 116 insertions(+), 2 deletions(-)

We are missing the BSD implementation.
Please add at least a function returning -ENOTSUP.

> +/**
> + * It unregisters the callback according to the specified interrupt handle,
> + * after it's no longer acive. Failes if source is not active.

Suggested reword:
Unregister the callback according to the specified interrupt handle,
after it's no longer active. Fail if source is not active.

> + *
> + * @param intr_handle
> + *  pointer to the interrupt handle.
> + * @param cb

The parameter is cb_fn.
Please check doxygen with "make doc-api-html"

> + *  callback address.
> + * @param cb_arg
> + *  address of parameter for callback, (void *)-1 means to remove all
> + *  registered which has the same callback address.
> + * @param ucb_fn
> + *  callback to call before cb is unregistered (optional).
> + *  can be used to close fd and free cb_arg.
> + *
> + * @return
> + *  - On success, return the number of callback entities marked for remove.
> + *  - On failure, a negative value.
> + */
> +int __rte_experimental
> +rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
> +				rte_intr_callback_fn cb_fn, void *cb_arg,
> +				rte_intr_unregister_callback_fn ucb_fn);

[dpdk-dev] [PATCH v9] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 lib/librte_eal/bsdapp/eal/eal_interrupts.c    | 100 ++++++++++++++++++
 .../common/include/rte_interrupts.h           |  32 ++++++
 lib/librte_eal/linuxapp/eal/eal_interrupts.c  |  85 ++++++++++++++-
 lib/librte_eal/rte_eal_version.map            |   1 +
 4 files changed, 216 insertions(+), 2 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

v9:
- add BSD implementation
- fix doxygen comments

diff --git a/lib/librte_eal/bsdapp/eal/eal_interrupts.c b/lib/librte_eal/bsdapp/eal/eal_interrupts.c
index 2feee2d52..45c5f67be 100644
--- a/lib/librte_eal/bsdapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/bsdapp/eal/eal_interrupts.c
@@ -33,6 +33,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -104,6 +106,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -189,6 +193,62 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -336,6 +396,7 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 	struct rte_intr_source *src;
 	bool call = false;
 	int n, bytes_read;
+	struct kevent ke;
 
 	for (n = 0; n < nfds; n++) {
 		int event_fd = events[n].ident;
@@ -415,6 +476,45 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				/* remove it from the kqueue */
+				memset(&ke, 0, sizeof(ke));
+				ke.flags = EV_DELETE; /* mark for deletion from the queue */
+
+				if (intr_source_to_kevent(src->intr_handle, &ke) < 0) {
+					RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+					rte_spinlock_unlock(&intr_lock);
+					return;
+				}
+
+				/**
+				 * remove intr file descriptor from wait list.
+				 */
+				if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+					RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
+						src->intr_handle.fd, strerror(errno));
+					/* removing non-existent even is an expected condition
+					 * in some circumstances (e.g. oneshot events).
+					 */
+				}
+
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 }
diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..225dae283 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * Unregister the callback according to the specified interrupt handle,
+ * after it's no longer active. Fail if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 3fe78260d..989e4d72a 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -318,6 +318,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

[dpdk-dev] [PATCH v9] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 lib/librte_eal/bsdapp/eal/eal_interrupts.c    | 100 ++++++++++++++++++
 .../common/include/rte_interrupts.h           |  32 ++++++
 lib/librte_eal/linuxapp/eal/eal_interrupts.c  |  85 ++++++++++++++-
 lib/librte_eal/rte_eal_version.map            |   1 +
 4 files changed, 216 insertions(+), 2 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

v9:
- add BSD implementation
- fix doxygen comments

diff --git a/lib/librte_eal/bsdapp/eal/eal_interrupts.c b/lib/librte_eal/bsdapp/eal/eal_interrupts.c
index 2feee2d52..45c5f67be 100644
--- a/lib/librte_eal/bsdapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/bsdapp/eal/eal_interrupts.c
@@ -33,6 +33,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -104,6 +106,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -189,6 +193,62 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -336,6 +396,7 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 	struct rte_intr_source *src;
 	bool call = false;
 	int n, bytes_read;
+	struct kevent ke;
 
 	for (n = 0; n < nfds; n++) {
 		int event_fd = events[n].ident;
@@ -415,6 +476,45 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				/* remove it from the kqueue */
+				memset(&ke, 0, sizeof(ke));
+				ke.flags = EV_DELETE; /* mark for deletion from the queue */
+
+				if (intr_source_to_kevent(src->intr_handle, &ke) < 0) {
+					RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+					rte_spinlock_unlock(&intr_lock);
+					return;
+				}
+
+				/**
+				 * remove intr file descriptor from wait list.
+				 */
+				if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+					RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
+						src->intr_handle.fd, strerror(errno));
+					/* removing non-existent even is an expected condition
+					 * in some circumstances (e.g. oneshot events).
+					 */
+				}
+
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 }
diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..225dae283 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * Unregister the callback according to the specified interrupt handle,
+ * after it's no longer active. Fail if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linuxapp/eal/eal_interrupts.c
+++ b/lib/librte_eal/linuxapp/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 3fe78260d..989e4d72a 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -318,6 +318,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

[dpdk-dev] [PATCH v10] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 .../common/include/rte_interrupts.h           |  32 ++++++
 lib/librte_eal/freebsd/eal/eal_interrupts.c   | 100 ++++++++++++++++++
 lib/librte_eal/linux/eal/eal_interrupts.c     |  85 ++++++++++++++-
 lib/librte_eal/rte_eal_version.map            |   1 +
 4 files changed, 216 insertions(+), 2 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

v9:
- add BSD implementation
- fix doxygen comments

v10:
- fix apply issues

diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..225dae283 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * Unregister the callback according to the specified interrupt handle,
+ * after it's no longer active. Fail if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/freebsd/eal/eal_interrupts.c b/lib/librte_eal/freebsd/eal/eal_interrupts.c
index 2feee2d52..45c5f67be 100644
--- a/lib/librte_eal/freebsd/eal/eal_interrupts.c
+++ b/lib/librte_eal/freebsd/eal/eal_interrupts.c
@@ -33,6 +33,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -104,6 +106,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -189,6 +193,62 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -336,6 +396,7 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 	struct rte_intr_source *src;
 	bool call = false;
 	int n, bytes_read;
+	struct kevent ke;
 
 	for (n = 0; n < nfds; n++) {
 		int event_fd = events[n].ident;
@@ -415,6 +476,45 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				/* remove it from the kqueue */
+				memset(&ke, 0, sizeof(ke));
+				ke.flags = EV_DELETE; /* mark for deletion from the queue */
+
+				if (intr_source_to_kevent(src->intr_handle, &ke) < 0) {
+					RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+					rte_spinlock_unlock(&intr_lock);
+					return;
+				}
+
+				/**
+				 * remove intr file descriptor from wait list.
+				 */
+				if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+					RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
+						src->intr_handle.fd, strerror(errno));
+					/* removing non-existent even is an expected condition
+					 * in some circumstances (e.g. oneshot events).
+					 */
+				}
+
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 }
diff --git a/lib/librte_eal/linux/eal/eal_interrupts.c b/lib/librte_eal/linux/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linux/eal/eal_interrupts.c
+++ b/lib/librte_eal/linux/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index eb5f7b9cb..bcfb16b32 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -322,6 +322,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

[dpdk-dev] [PATCH v10] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 .../common/include/rte_interrupts.h           |  32 ++++++
 lib/librte_eal/freebsd/eal/eal_interrupts.c   | 100 ++++++++++++++++++
 lib/librte_eal/linux/eal/eal_interrupts.c     |  85 ++++++++++++++-
 lib/librte_eal/rte_eal_version.map            |   1 +
 4 files changed, 216 insertions(+), 2 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

v9:
- add BSD implementation
- fix doxygen comments

v10:
- fix apply issues

diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..225dae283 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * Unregister the callback according to the specified interrupt handle,
+ * after it's no longer active. Fail if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/freebsd/eal/eal_interrupts.c b/lib/librte_eal/freebsd/eal/eal_interrupts.c
index 2feee2d52..45c5f67be 100644
--- a/lib/librte_eal/freebsd/eal/eal_interrupts.c
+++ b/lib/librte_eal/freebsd/eal/eal_interrupts.c
@@ -33,6 +33,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -104,6 +106,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -189,6 +193,62 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -336,6 +396,7 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 	struct rte_intr_source *src;
 	bool call = false;
 	int n, bytes_read;
+	struct kevent ke;
 
 	for (n = 0; n < nfds; n++) {
 		int event_fd = events[n].ident;
@@ -415,6 +476,45 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				/* remove it from the kqueue */
+				memset(&ke, 0, sizeof(ke));
+				ke.flags = EV_DELETE; /* mark for deletion from the queue */
+
+				if (intr_source_to_kevent(src->intr_handle, &ke) < 0) {
+					RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+					rte_spinlock_unlock(&intr_lock);
+					return;
+				}
+
+				/**
+				 * remove intr file descriptor from wait list.
+				 */
+				if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+					RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
+						src->intr_handle.fd, strerror(errno));
+					/* removing non-existent even is an expected condition
+					 * in some circumstances (e.g. oneshot events).
+					 */
+				}
+
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 }
diff --git a/lib/librte_eal/linux/eal/eal_interrupts.c b/lib/librte_eal/linux/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linux/eal/eal_interrupts.c
+++ b/lib/librte_eal/linux/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index eb5f7b9cb..bcfb16b32 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -322,6 +322,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

[dpdk-dev] [PATCH v11] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 .../common/include/rte_interrupts.h           |  32 ++++++
 lib/librte_eal/freebsd/eal/eal_interrupts.c   | 102 +++++++++++++++++-
 lib/librte_eal/linux/eal/eal_interrupts.c     |  85 ++++++++++++++-
 lib/librte_eal/rte_eal_version.map            |   1 +
 4 files changed, 217 insertions(+), 3 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

v9:
- add BSD implementation
- fix doxygen comments

v10:
- fix apply issues

v11:
- fix freebsd compilation issues

diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..225dae283 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * Unregister the callback according to the specified interrupt handle,
+ * after it's no longer active. Fail if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/freebsd/eal/eal_interrupts.c b/lib/librte_eal/freebsd/eal/eal_interrupts.c
index 2feee2d52..82964c1f5 100644
--- a/lib/librte_eal/freebsd/eal/eal_interrupts.c
+++ b/lib/librte_eal/freebsd/eal/eal_interrupts.c
@@ -33,6 +33,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -104,6 +106,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -189,6 +193,62 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -332,10 +392,11 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 {
 	struct rte_intr_callback active_cb;
 	union rte_intr_read_buffer buf;
-	struct rte_intr_callback *cb;
+	struct rte_intr_callback *cb, *next;
 	struct rte_intr_source *src;
 	bool call = false;
 	int n, bytes_read;
+	struct kevent ke;
 
 	for (n = 0; n < nfds; n++) {
 		int event_fd = events[n].ident;
@@ -415,6 +476,45 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				/* remove it from the kqueue */
+				memset(&ke, 0, sizeof(ke));
+				ke.flags = EV_DELETE; /* mark for deletion from the queue */
+
+				if (intr_source_to_kevent(&src->intr_handle, &ke) < 0) {
+					RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+					rte_spinlock_unlock(&intr_lock);
+					return;
+				}
+
+				/**
+				 * remove intr file descriptor from wait list.
+				 */
+				if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+					RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
+						src->intr_handle.fd, strerror(errno));
+					/* removing non-existent even is an expected condition
+					 * in some circumstances (e.g. oneshot events).
+					 */
+				}
+
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 }
diff --git a/lib/librte_eal/linux/eal/eal_interrupts.c b/lib/librte_eal/linux/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linux/eal/eal_interrupts.c
+++ b/lib/librte_eal/linux/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index eb5f7b9cb..bcfb16b32 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -322,6 +322,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

[dpdk-dev] [PATCH v11] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 .../common/include/rte_interrupts.h           |  32 ++++++
 lib/librte_eal/freebsd/eal/eal_interrupts.c   | 102 +++++++++++++++++-
 lib/librte_eal/linux/eal/eal_interrupts.c     |  85 ++++++++++++++-
 lib/librte_eal/rte_eal_version.map            |   1 +
 4 files changed, 217 insertions(+), 3 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

v9:
- add BSD implementation
- fix doxygen comments

v10:
- fix apply issues

v11:
- fix freebsd compilation issues

diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..225dae283 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * Unregister the callback according to the specified interrupt handle,
+ * after it's no longer active. Fail if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/freebsd/eal/eal_interrupts.c b/lib/librte_eal/freebsd/eal/eal_interrupts.c
index 2feee2d52..82964c1f5 100644
--- a/lib/librte_eal/freebsd/eal/eal_interrupts.c
+++ b/lib/librte_eal/freebsd/eal/eal_interrupts.c
@@ -33,6 +33,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -104,6 +106,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -189,6 +193,62 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -332,10 +392,11 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 {
 	struct rte_intr_callback active_cb;
 	union rte_intr_read_buffer buf;
-	struct rte_intr_callback *cb;
+	struct rte_intr_callback *cb, *next;
 	struct rte_intr_source *src;
 	bool call = false;
 	int n, bytes_read;
+	struct kevent ke;
 
 	for (n = 0; n < nfds; n++) {
 		int event_fd = events[n].ident;
@@ -415,6 +476,45 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				/* remove it from the kqueue */
+				memset(&ke, 0, sizeof(ke));
+				ke.flags = EV_DELETE; /* mark for deletion from the queue */
+
+				if (intr_source_to_kevent(&src->intr_handle, &ke) < 0) {
+					RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+					rte_spinlock_unlock(&intr_lock);
+					return;
+				}
+
+				/**
+				 * remove intr file descriptor from wait list.
+				 */
+				if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+					RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
+						src->intr_handle.fd, strerror(errno));
+					/* removing non-existent even is an expected condition
+					 * in some circumstances (e.g. oneshot events).
+					 */
+				}
+
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 }
diff --git a/lib/librte_eal/linux/eal/eal_interrupts.c b/lib/librte_eal/linux/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linux/eal/eal_interrupts.c
+++ b/lib/librte_eal/linux/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index eb5f7b9cb..bcfb16b32 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -322,6 +322,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

[dpdk-dev] [PATCH v12] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 .../common/include/rte_interrupts.h           |  32 ++++++
 lib/librte_eal/freebsd/eal/eal_interrupts.c   | 105 +++++++++++++++++-
 lib/librte_eal/linux/eal/eal_interrupts.c     |  85 +++++++++++++-
 lib/librte_eal/rte_eal_version.map            |   1 +
 4 files changed, 220 insertions(+), 3 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

v9:
- add BSD implementation
- fix doxygen comments

v10:
- fix apply issues

v11:
- fix freebsd compilation issues

diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..225dae283 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * Unregister the callback according to the specified interrupt handle,
+ * after it's no longer active. Fail if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/freebsd/eal/eal_interrupts.c b/lib/librte_eal/freebsd/eal/eal_interrupts.c
index 2feee2d52..4a9aedd11 100644
--- a/lib/librte_eal/freebsd/eal/eal_interrupts.c
+++ b/lib/librte_eal/freebsd/eal/eal_interrupts.c
@@ -33,6 +33,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -104,6 +106,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -189,6 +193,62 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -332,10 +392,11 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 {
 	struct rte_intr_callback active_cb;
 	union rte_intr_read_buffer buf;
-	struct rte_intr_callback *cb;
+	struct rte_intr_callback *cb, *next;
 	struct rte_intr_source *src;
 	bool call = false;
 	int n, bytes_read;
+	struct kevent ke;
 
 	for (n = 0; n < nfds; n++) {
 		int event_fd = events[n].ident;
@@ -415,6 +476,48 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				/* remove it from the kqueue */
+				memset(&ke, 0, sizeof(ke));
+				/* mark for deletion from the queue */
+				ke.flags = EV_DELETE;
+
+				if (intr_source_to_kevent(&src->intr_handle, &ke) < 0) {
+					RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+					rte_spinlock_unlock(&intr_lock);
+					return;
+				}
+
+				/**
+				 * remove intr file descriptor from wait list.
+				 */
+				if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+					RTE_LOG(ERR, EAL, "Error removing fd %d kevent, "
+						"%s\n", src->intr_handle.fd,
+						strerror(errno));
+					/* removing non-existent even is an expected
+					 * condition in some circumstances
+					 * (e.g. oneshot events).
+					 */
+				}
+
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 }
diff --git a/lib/librte_eal/linux/eal/eal_interrupts.c b/lib/librte_eal/linux/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linux/eal/eal_interrupts.c
+++ b/lib/librte_eal/linux/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index eb5f7b9cb..bcfb16b32 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -322,6 +322,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

[dpdk-dev] [PATCH v12] eal_interrupts: add option for pending callback unregister

[Jakub Grajciar]

use case: if callback is used to receive message form socket,
and the message received is disconnect/error, this callback needs
to be unregistered, but cannot because it is still active.

With this patch it is possible to mark the callback to be
unregistered once the interrupt process is done with this
interrupt source.

Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>
---
 .../common/include/rte_interrupts.h           |  32 ++++++
 lib/librte_eal/freebsd/eal/eal_interrupts.c   | 105 +++++++++++++++++-
 lib/librte_eal/linux/eal/eal_interrupts.c     |  85 +++++++++++++-
 lib/librte_eal/rte_eal_version.map            |   1 +
 4 files changed, 220 insertions(+), 3 deletions(-)

v2:
- fix coding syle

v3:
- fix locks

v4:
- update version map

v6:
- add _rte_experimental to rte_intr_callback_unregister_pending

v8:
- fix compilation errors

v9:
- add BSD implementation
- fix doxygen comments

v10:
- fix apply issues

v11:
- fix freebsd compilation issues

diff --git a/lib/librte_eal/common/include/rte_interrupts.h b/lib/librte_eal/common/include/rte_interrupts.h
index d751a6378..225dae283 100644
--- a/lib/librte_eal/common/include/rte_interrupts.h
+++ b/lib/librte_eal/common/include/rte_interrupts.h
@@ -6,6 +6,7 @@
 #define _RTE_INTERRUPTS_H_
 
 #include <rte_common.h>
+#include <rte_compat.h>
 
 /**
  * @file
@@ -24,6 +25,13 @@ struct rte_intr_handle;
 /** Function to be registered for the specific interrupt */
 typedef void (*rte_intr_callback_fn)(void *cb_arg);
 
+/**
+ * Function to call after a callback is unregistered.
+ * Can be used to close fd and free cb_arg.
+ */
+typedef void (*rte_intr_unregister_callback_fn)(struct rte_intr_handle *intr_handle,
+						void *cb_arg);
+
 #include "rte_eal_interrupts.h"
 
 /**
@@ -61,6 +69,30 @@ int rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 int rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 				rte_intr_callback_fn cb, void *cb_arg);
 
+/**
+ * Unregister the callback according to the specified interrupt handle,
+ * after it's no longer active. Fail if source is not active.
+ *
+ * @param intr_handle
+ *  pointer to the interrupt handle.
+ * @param cb_fn
+ *  callback address.
+ * @param cb_arg
+ *  address of parameter for callback, (void *)-1 means to remove all
+ *  registered which has the same callback address.
+ * @param ucb_fn
+ *  callback to call before cb is unregistered (optional).
+ *  can be used to close fd and free cb_arg.
+ *
+ * @return
+ *  - On success, return the number of callback entities marked for remove.
+ *  - On failure, a negative value.
+ */
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn);
+
 /**
  * It enables the interrupt for the specified handle.
  *
diff --git a/lib/librte_eal/freebsd/eal/eal_interrupts.c b/lib/librte_eal/freebsd/eal/eal_interrupts.c
index 2feee2d52..4a9aedd11 100644
--- a/lib/librte_eal/freebsd/eal/eal_interrupts.c
+++ b/lib/librte_eal/freebsd/eal/eal_interrupts.c
@@ -33,6 +33,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -104,6 +106,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -189,6 +193,62 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	if (kq < 0) {
+		RTE_LOG(ERR, EAL, "Kqueue is not active\n");
+		return -ENODEV;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 		rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -332,10 +392,11 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 {
 	struct rte_intr_callback active_cb;
 	union rte_intr_read_buffer buf;
-	struct rte_intr_callback *cb;
+	struct rte_intr_callback *cb, *next;
 	struct rte_intr_source *src;
 	bool call = false;
 	int n, bytes_read;
+	struct kevent ke;
 
 	for (n = 0; n < nfds; n++) {
 		int event_fd = events[n].ident;
@@ -415,6 +476,48 @@ eal_intr_process_interrupts(struct kevent *events, int nfds)
 
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				/* remove it from the kqueue */
+				memset(&ke, 0, sizeof(ke));
+				/* mark for deletion from the queue */
+				ke.flags = EV_DELETE;
+
+				if (intr_source_to_kevent(&src->intr_handle, &ke) < 0) {
+					RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
+					rte_spinlock_unlock(&intr_lock);
+					return;
+				}
+
+				/**
+				 * remove intr file descriptor from wait list.
+				 */
+				if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
+					RTE_LOG(ERR, EAL, "Error removing fd %d kevent, "
+						"%s\n", src->intr_handle.fd,
+						strerror(errno));
+					/* removing non-existent even is an expected
+					 * condition in some circumstances
+					 * (e.g. oneshot events).
+					 */
+				}
+
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 }
diff --git a/lib/librte_eal/linux/eal/eal_interrupts.c b/lib/librte_eal/linux/eal/eal_interrupts.c
index cbac451e1..be0dd4490 100644
--- a/lib/librte_eal/linux/eal/eal_interrupts.c
+++ b/lib/librte_eal/linux/eal/eal_interrupts.c
@@ -76,6 +76,8 @@ struct rte_intr_callback {
 	TAILQ_ENTRY(rte_intr_callback) next;
 	rte_intr_callback_fn cb_fn;  /**< callback address */
 	void *cb_arg;                /**< parameter for callback */
+	uint8_t pending_delete;      /**< delete after callback is called */
+	rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
 };
 
 struct rte_intr_source {
@@ -472,6 +474,8 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	}
 	callback->cb_fn = cb;
 	callback->cb_arg = cb_arg;
+	callback->pending_delete = 0;
+	callback->ucb_fn = NULL;
 
 	rte_spinlock_lock(&intr_lock);
 
@@ -518,6 +522,57 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 	return ret;
 }
 
+int __rte_experimental
+rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
+				rte_intr_callback_fn cb_fn, void *cb_arg,
+				rte_intr_unregister_callback_fn ucb_fn)
+{
+	int ret;
+	struct rte_intr_source *src;
+	struct rte_intr_callback *cb, *next;
+
+	/* do parameter checking first */
+	if (intr_handle == NULL || intr_handle->fd < 0) {
+		RTE_LOG(ERR, EAL,
+		"Unregistering with invalid input parameter\n");
+		return -EINVAL;
+	}
+
+	rte_spinlock_lock(&intr_lock);
+
+	/* check if the insterrupt source for the fd is existent */
+	TAILQ_FOREACH(src, &intr_sources, next)
+		if (src->intr_handle.fd == intr_handle->fd)
+			break;
+
+	/* No interrupt source registered for the fd */
+	if (src == NULL) {
+		ret = -ENOENT;
+
+	/* only usable if the source is active */
+	} else if (src->active == 0) {
+		ret = -EAGAIN;
+
+	} else {
+		ret = 0;
+
+		/* walk through the callbacks and mark all that match. */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
+					cb->cb_arg == cb_arg)) {
+				cb->pending_delete = 1;
+				cb->ucb_fn = ucb_fn;
+				ret++;
+			}
+		}
+	}
+
+	rte_spinlock_unlock(&intr_lock);
+
+	return ret;
+}
+
 int
 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 			rte_intr_callback_fn cb_fn, void *cb_arg)
@@ -698,7 +753,7 @@ static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
 	bool call = false;
-	int n, bytes_read;
+	int n, bytes_read, rv;
 	struct rte_intr_source *src;
 	struct rte_intr_callback *cb, *next;
 	union rte_intr_read_buffer buf;
@@ -823,9 +878,35 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 				rte_spinlock_lock(&intr_lock);
 			}
 		}
-
 		/* we done with that interrupt source, release it. */
 		src->active = 0;
+
+		rv = 0;
+
+		/* check if any callback are supposed to be removed */
+		for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
+			next = TAILQ_NEXT(cb, next);
+			if (cb->pending_delete) {
+				TAILQ_REMOVE(&src->callbacks, cb, next);
+				if (cb->ucb_fn)
+					cb->ucb_fn(&src->intr_handle, cb->cb_arg);
+				free(cb);
+				rv++;
+			}
+		}
+
+		/* all callbacks for that source are removed. */
+		if (TAILQ_EMPTY(&src->callbacks)) {
+			TAILQ_REMOVE(&intr_sources, src, next);
+			free(src);
+		}
+
+		/* notify the pipe fd waited by epoll_wait to rebuild the wait list */
+		if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
+			rte_spinlock_unlock(&intr_lock);
+			return -EPIPE;
+		}
+
 		rte_spinlock_unlock(&intr_lock);
 	}
 
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index eb5f7b9cb..bcfb16b32 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -322,6 +322,7 @@ EXPERIMENTAL {
 	rte_fbarray_is_used;
 	rte_fbarray_set_free;
 	rte_fbarray_set_used;
+	rte_intr_callback_unregister_pending;
 	rte_log_register_type_and_pick_level;
 	rte_malloc_dump_heaps;
 	rte_malloc_heap_create;
-- 
2.17.1

Re: [dpdk-dev] [PATCH v12] eal_interrupts: add option for pending callback unregister

[Thomas Monjalon]

22/03/2019 12:52, Jakub Grajciar:
> use case: if callback is used to receive message form socket,
> and the message received is disconnect/error, this callback needs
> to be unregistered, but cannot because it is still active.
> 
> With this patch it is possible to mark the callback to be
> unregistered once the interrupt process is done with this
> interrupt source.
> 
> Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>

Applied, thanks

Re: [dpdk-dev] [PATCH v12] eal_interrupts: add option for pending callback unregister

[Thomas Monjalon]

22/03/2019 12:52, Jakub Grajciar:
> use case: if callback is used to receive message form socket,
> and the message received is disconnect/error, this callback needs
> to be unregistered, but cannot because it is still active.
> 
> With this patch it is possible to mark the callback to be
> unregistered once the interrupt process is done with this
> interrupt source.
> 
> Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>

Applied, thanks