DPDK patches and discussions
 help / color / mirror / Atom feed
From: <eagostini@nvidia.com>
To: <dev@dpdk.org>
Cc: Thomas Monjalon <thomas@monjalon.net>
Subject: [dpdk-dev] [PATCH v3 2/9] gpudev: add event notification
Date: Sat, 9 Oct 2021 01:53:42 +0000	[thread overview]
Message-ID: <20211009015349.9694-3-eagostini@nvidia.com> (raw)
In-Reply-To: <20211009015349.9694-1-eagostini@nvidia.com>

From: Thomas Monjalon <thomas@monjalon.net>

Callback functions may be registered for a device event.
Callback management is per-process and not thread-safe.

The events RTE_GPU_EVENT_NEW and RTE_GPU_EVENT_DEL
are notified respectively after creation and before removal
of a device, as part of the library functions.
Some future events may be emitted from drivers.

Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
---
 lib/gpudev/gpudev.c        | 148 +++++++++++++++++++++++++++++++++++++
 lib/gpudev/gpudev_driver.h |   7 ++
 lib/gpudev/rte_gpudev.h    |  70 ++++++++++++++++++
 lib/gpudev/version.map     |   3 +
 4 files changed, 228 insertions(+)

diff --git a/lib/gpudev/gpudev.c b/lib/gpudev/gpudev.c
index c839c530c8..d57e23df7c 100644
--- a/lib/gpudev/gpudev.c
+++ b/lib/gpudev/gpudev.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_eal.h>
+#include <rte_tailq.h>
 #include <rte_string_fns.h>
 #include <rte_errno.h>
 #include <rte_log.h>
@@ -27,6 +28,16 @@ static int16_t gpu_max;
 /* Number of currently valid devices */
 static int16_t gpu_count;
 
+/* Event callback object */
+struct rte_gpu_callback {
+	TAILQ_ENTRY(rte_gpu_callback) next;
+	rte_gpu_callback_t *function;
+	void *user_data;
+	enum rte_gpu_event event;
+};
+static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER;
+static void gpu_free_callbacks(struct rte_gpu *dev);
+
 int
 rte_gpu_init(size_t dev_max)
 {
@@ -166,6 +177,7 @@ rte_gpu_allocate(const char *name)
 	dev->info.name = dev->name;
 	dev->info.dev_id = dev_id;
 	dev->info.numa_node = -1;
+	TAILQ_INIT(&dev->callbacks);
 
 	gpu_count++;
 	GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
@@ -180,6 +192,8 @@ rte_gpu_complete_new(struct rte_gpu *dev)
 		return;
 
 	dev->state = RTE_GPU_STATE_INITIALIZED;
+	dev->state = RTE_GPU_STATE_INITIALIZED;
+	rte_gpu_notify(dev, RTE_GPU_EVENT_NEW);
 }
 
 int
@@ -192,6 +206,9 @@ rte_gpu_release(struct rte_gpu *dev)
 
 	GPU_LOG(DEBUG, "free device %s (id %d)",
 			dev->info.name, dev->info.dev_id);
+	rte_gpu_notify(dev, RTE_GPU_EVENT_DEL);
+
+	gpu_free_callbacks(dev);
 	dev->state = RTE_GPU_STATE_UNUSED;
 	gpu_count--;
 
@@ -224,6 +241,137 @@ rte_gpu_close(int16_t dev_id)
 	return firsterr;
 }
 
+int
+rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event,
+		rte_gpu_callback_t *function, void *user_data)
+{
+	int16_t next_dev, last_dev;
+	struct rte_gpu_callback_list *callbacks;
+	struct rte_gpu_callback *callback;
+
+	if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
+		GPU_LOG(ERR, "register callback of invalid ID %d", dev_id);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (function == NULL) {
+		GPU_LOG(ERR, "cannot register callback without function");
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+
+	if (dev_id == RTE_GPU_ID_ANY) {
+		next_dev = 0;
+		last_dev = gpu_max - 1;
+	} else {
+		next_dev = last_dev = dev_id;
+	}
+
+	rte_rwlock_write_lock(&gpu_callback_lock);
+	do {
+		callbacks = &gpus[next_dev].callbacks;
+
+		/* check if not already registered */
+		TAILQ_FOREACH(callback, callbacks, next) {
+			if (callback->event == event &&
+					callback->function == function &&
+					callback->user_data == user_data) {
+				GPU_LOG(INFO, "callback already registered");
+				return 0;
+			}
+		}
+
+		callback = malloc(sizeof(*callback));
+		if (callback == NULL) {
+			GPU_LOG(ERR, "cannot allocate callback");
+			return -ENOMEM;
+		}
+		callback->function = function;
+		callback->user_data = user_data;
+		callback->event = event;
+		TAILQ_INSERT_TAIL(callbacks, callback, next);
+
+	} while (++next_dev <= last_dev);
+	rte_rwlock_write_unlock(&gpu_callback_lock);
+
+	return 0;
+}
+
+int
+rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event,
+		rte_gpu_callback_t *function, void *user_data)
+{
+	int16_t next_dev, last_dev;
+	struct rte_gpu_callback_list *callbacks;
+	struct rte_gpu_callback *callback, *nextcb;
+
+	if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
+		GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (function == NULL) {
+		GPU_LOG(ERR, "cannot unregister callback without function");
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+
+	if (dev_id == RTE_GPU_ID_ANY) {
+		next_dev = 0;
+		last_dev = gpu_max - 1;
+	} else {
+		next_dev = last_dev = dev_id;
+	}
+
+	rte_rwlock_write_lock(&gpu_callback_lock);
+	do {
+		callbacks = &gpus[next_dev].callbacks;
+		RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
+			if (callback->event != event ||
+					callback->function != function ||
+					(callback->user_data != user_data &&
+					user_data != (void *)-1))
+				continue;
+			TAILQ_REMOVE(callbacks, callback, next);
+			free(callback);
+		}
+	} while (++next_dev <= last_dev);
+	rte_rwlock_write_unlock(&gpu_callback_lock);
+
+	return 0;
+}
+
+static void
+gpu_free_callbacks(struct rte_gpu *dev)
+{
+	struct rte_gpu_callback_list *callbacks;
+	struct rte_gpu_callback *callback, *nextcb;
+
+	callbacks = &dev->callbacks;
+	rte_rwlock_write_lock(&gpu_callback_lock);
+	RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
+		TAILQ_REMOVE(callbacks, callback, next);
+		free(callback);
+	}
+	rte_rwlock_write_unlock(&gpu_callback_lock);
+}
+
+void
+rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event)
+{
+	int16_t dev_id;
+	struct rte_gpu_callback *callback;
+
+	dev_id = dev->info.dev_id;
+	rte_rwlock_read_lock(&gpu_callback_lock);
+	TAILQ_FOREACH(callback, &dev->callbacks, next) {
+		if (callback->event != event || callback->function == NULL)
+			continue;
+		callback->function(dev_id, event, callback->user_data);
+	}
+	rte_rwlock_read_unlock(&gpu_callback_lock);
+}
+
 int
 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info)
 {
diff --git a/lib/gpudev/gpudev_driver.h b/lib/gpudev/gpudev_driver.h
index 9e096e3b64..2a7089aa52 100644
--- a/lib/gpudev/gpudev_driver.h
+++ b/lib/gpudev/gpudev_driver.h
@@ -12,6 +12,7 @@
 #define RTE_GPUDEV_DRIVER_H
 
 #include <stdint.h>
+#include <sys/queue.h>
 
 #include <rte_dev.h>
 
@@ -43,6 +44,8 @@ struct rte_gpu {
 	struct rte_gpu_info info;
 	/* Driver functions. */
 	struct rte_gpu_ops ops;
+	/* Event callback list. */
+	TAILQ_HEAD(rte_gpu_callback_list, rte_gpu_callback) callbacks;
 	/* Current state (used or not) in the running process. */
 	enum rte_gpu_state state; /* Updated by this library. */
 	/* Driver-specific private data for the running process. */
@@ -64,4 +67,8 @@ void rte_gpu_complete_new(struct rte_gpu *dev);
 __rte_internal
 int rte_gpu_release(struct rte_gpu *dev);
 
+/* Call registered callbacks. No multi-process event. */
+__rte_internal
+void rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event);
+
 #endif /* RTE_GPUDEV_DRIVER_H */
diff --git a/lib/gpudev/rte_gpudev.h b/lib/gpudev/rte_gpudev.h
index eb7cfa8c59..e1702fbfe4 100644
--- a/lib/gpudev/rte_gpudev.h
+++ b/lib/gpudev/rte_gpudev.h
@@ -31,6 +31,11 @@ extern "C" {
 
 /** Empty device ID. */
 #define RTE_GPU_ID_NONE -1
+/** Catch-all device ID. */
+#define RTE_GPU_ID_ANY INT16_MIN
+
+/** Catch-all callback data. */
+#define RTE_GPU_CALLBACK_ANY_DATA ((void *)-1)
 
 /** Store device info. */
 struct rte_gpu_info {
@@ -46,6 +51,18 @@ struct rte_gpu_info {
 	int16_t numa_node;
 };
 
+/** Flags passed in notification callback. */
+enum rte_gpu_event {
+	/** Device is just initialized. */
+	RTE_GPU_EVENT_NEW,
+	/** Device is going to be released. */
+	RTE_GPU_EVENT_DEL,
+};
+
+/** Prototype of event callback function. */
+typedef void (rte_gpu_callback_t)(int16_t dev_id,
+		enum rte_gpu_event event, void *user_data);
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -141,6 +158,59 @@ int16_t rte_gpu_find_next(int16_t dev_id);
 __rte_experimental
 int rte_gpu_close(int16_t dev_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Register a function as event callback.
+ * A function may be registered multiple times for different events.
+ *
+ * @param dev_id
+ *   Device ID to get notified about.
+ *   RTE_GPU_ID_ANY means all devices.
+ * @param event
+ *   Device event to be registered for.
+ * @param function
+ *   Callback function to be called on event.
+ * @param user_data
+ *   Optional parameter passed in the callback.
+ *
+ * @return
+ *   0 on success, -rte_errno otherwise:
+ *   - ENODEV if invalid dev_id
+ *   - EINVAL if NULL function
+ *   - ENOMEM if out of memory
+ */
+__rte_experimental
+int rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event,
+		rte_gpu_callback_t *function, void *user_data);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Unregister for an event.
+ *
+ * @param dev_id
+ *   Device ID to be silenced.
+ *   RTE_GPU_ID_ANY means all devices.
+ * @param event
+ *   Registered event.
+ * @param function
+ *   Registered function.
+ * @param user_data
+ *   Optional parameter as registered.
+ *   RTE_GPU_CALLBACK_ANY_DATA is a catch-all.
+ *
+ * @return
+ *   0 on success, -rte_errno otherwise:
+ *   - ENODEV if invalid dev_id
+ *   - EINVAL if NULL function
+ */
+__rte_experimental
+int rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event,
+		rte_gpu_callback_t *function, void *user_data);
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
diff --git a/lib/gpudev/version.map b/lib/gpudev/version.map
index 6ac6b327e2..b3b6b76c1c 100644
--- a/lib/gpudev/version.map
+++ b/lib/gpudev/version.map
@@ -2,6 +2,8 @@ EXPERIMENTAL {
 	global:
 
 	# added in 21.11
+	rte_gpu_callback_register;
+	rte_gpu_callback_unregister;
 	rte_gpu_close;
 	rte_gpu_count_avail;
 	rte_gpu_find_next;
@@ -16,5 +18,6 @@ INTERNAL {
 	rte_gpu_allocate;
 	rte_gpu_complete_new;
 	rte_gpu_get_by_name;
+	rte_gpu_notify;
 	rte_gpu_release;
 };
-- 
2.17.1


  parent reply	other threads:[~2021-10-08 17:44 UTC|newest]

Thread overview: 128+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-06-02 20:35 [dpdk-dev] [PATCH] gpudev: introduce memory API Thomas Monjalon
2021-06-02 20:46 ` Stephen Hemminger
2021-06-02 20:48   ` Thomas Monjalon
2021-06-03  7:06 ` Andrew Rybchenko
2021-06-03  7:26   ` Thomas Monjalon
2021-06-03  7:49     ` Andrew Rybchenko
2021-06-03  8:26       ` Thomas Monjalon
2021-06-03  8:57         ` Andrew Rybchenko
2021-06-03  7:18 ` David Marchand
2021-06-03  7:30   ` Thomas Monjalon
2021-06-03  7:47 ` Jerin Jacob
2021-06-03  8:28   ` Thomas Monjalon
2021-06-03  8:41     ` Jerin Jacob
2021-06-03  8:43       ` Thomas Monjalon
2021-06-03  8:47         ` Jerin Jacob
2021-06-03  8:53           ` Thomas Monjalon
2021-06-03  9:20             ` Jerin Jacob
2021-06-03  9:36               ` Thomas Monjalon
2021-06-03 10:04                 ` Jerin Jacob
2021-06-03 10:30                   ` Thomas Monjalon
2021-06-03 11:38                     ` Jerin Jacob
2021-06-04 12:55                       ` Thomas Monjalon
2021-06-04 15:05                         ` Jerin Jacob
2021-06-03  9:33   ` Ferruh Yigit
2021-06-04 10:28     ` Thomas Monjalon
2021-06-04 11:09       ` Jerin Jacob
2021-06-04 12:46         ` Thomas Monjalon
2021-06-04 13:05           ` Andrew Rybchenko
2021-06-04 13:18             ` Thomas Monjalon
2021-06-04 13:59               ` Andrew Rybchenko
2021-06-04 14:09                 ` Thomas Monjalon
2021-06-04 15:20                   ` Jerin Jacob
2021-06-04 15:51                     ` Thomas Monjalon
2021-06-04 18:20                       ` Wang, Haiyue
2021-06-05  5:09                         ` Jerin Jacob
2021-06-06  1:13                           ` Honnappa Nagarahalli
2021-06-06  5:28                             ` Jerin Jacob
2021-06-07 10:29                               ` Thomas Monjalon
2021-06-07  7:20                             ` Wang, Haiyue
2021-06-07 10:43                               ` Thomas Monjalon
2021-06-07 13:54                                 ` Jerin Jacob
2021-06-07 16:47                                   ` Thomas Monjalon
2021-06-08  4:10                                     ` Jerin Jacob
2021-06-08  6:34                                       ` Thomas Monjalon
2021-06-08  7:09                                         ` Jerin Jacob
2021-06-08  7:32                                           ` Thomas Monjalon
2021-06-15 18:24                                         ` Ferruh Yigit
2021-06-15 18:54                                           ` Thomas Monjalon
2021-06-07 23:31                                   ` Honnappa Nagarahalli
2021-06-04  5:51 ` Wang, Haiyue
2021-06-04  8:15   ` Thomas Monjalon
2021-06-04 11:07 ` Wang, Haiyue
2021-06-04 12:43   ` Thomas Monjalon
2021-06-04 13:25     ` Wang, Haiyue
2021-06-04 14:06       ` Thomas Monjalon
2021-06-04 18:04         ` Wang, Haiyue
2021-06-05  7:49           ` Thomas Monjalon
2021-06-05 11:09             ` Wang, Haiyue
2021-06-06  1:10 ` Honnappa Nagarahalli
2021-06-07 10:50   ` Thomas Monjalon
2021-07-30 13:55 ` [dpdk-dev] [RFC PATCH v2 0/7] heterogeneous computing library Thomas Monjalon
2021-07-30 13:55   ` [dpdk-dev] [RFC PATCH v2 1/7] hcdev: introduce heterogeneous computing device library Thomas Monjalon
2021-07-30 13:55   ` [dpdk-dev] [RFC PATCH v2 2/7] hcdev: add event notification Thomas Monjalon
2021-07-30 13:55   ` [dpdk-dev] [RFC PATCH v2 3/7] hcdev: add child device representing a device context Thomas Monjalon
2021-07-30 13:55   ` [dpdk-dev] [RFC PATCH v2 4/7] hcdev: support multi-process Thomas Monjalon
2021-07-30 13:55   ` [dpdk-dev] [RFC PATCH v2 5/7] hcdev: add memory API Thomas Monjalon
2021-07-30 13:55   ` [dpdk-dev] [RFC PATCH v2 6/7] hcdev: add communication flag Thomas Monjalon
2021-07-30 13:55   ` [dpdk-dev] [RFC PATCH v2 7/7] hcdev: add communication list Thomas Monjalon
2021-07-31  7:06   ` [dpdk-dev] [RFC PATCH v2 0/7] heterogeneous computing library Jerin Jacob
2021-07-31  8:21     ` Thomas Monjalon
2021-07-31 13:42       ` Jerin Jacob
2021-08-27  9:44         ` Thomas Monjalon
2021-08-27 12:19           ` Jerin Jacob
2021-08-29  5:32             ` Wang, Haiyue
2021-09-01 15:35               ` Elena Agostini
2021-09-02 13:12                 ` Jerin Jacob
2021-09-06 16:11                   ` Elena Agostini
2021-09-06 17:15                     ` Wang, Haiyue
2021-09-06 17:22                       ` Elena Agostini
2021-09-07  0:55                         ` Wang, Haiyue
2021-10-09  1:53 ` [dpdk-dev] [PATCH v3 0/9] GPU library eagostini
2021-10-09  1:53   ` [dpdk-dev] [PATCH v3 1/9] gpudev: introduce GPU device class library eagostini
2021-10-09  1:53   ` eagostini [this message]
2021-10-09  1:53   ` [dpdk-dev] [PATCH v3 3/9] gpudev: add child device representing a device context eagostini
2021-10-09  1:53   ` [dpdk-dev] [PATCH v3 4/9] gpudev: support multi-process eagostini
2021-10-09  1:53   ` [dpdk-dev] [PATCH v3 5/9] gpudev: add memory API eagostini
2021-10-08 20:18     ` Thomas Monjalon
2021-10-29 19:38     ` Mattias Rönnblom
2021-11-08 15:16       ` Elena Agostini
2021-10-09  1:53   ` [dpdk-dev] [PATCH v3 6/9] gpudev: add memory barrier eagostini
2021-10-08 20:16     ` Thomas Monjalon
2021-10-09  1:53   ` [dpdk-dev] [PATCH v3 7/9] gpudev: add communication flag eagostini
2021-10-09  1:53   ` [dpdk-dev] [PATCH v3 8/9] gpudev: add communication list eagostini
2021-10-09  1:53   ` [dpdk-dev] [PATCH v3 9/9] doc: add CUDA example in GPU guide eagostini
2021-10-10 10:16   ` [dpdk-dev] [PATCH v3 0/9] GPU library Jerin Jacob
2021-10-11  8:18     ` Thomas Monjalon
2021-10-11  8:43       ` Jerin Jacob
2021-10-11  9:12         ` Thomas Monjalon
2021-10-11  9:29           ` Jerin Jacob
2021-10-11 10:27             ` Thomas Monjalon
2021-10-11 11:41               ` Jerin Jacob
2021-10-11 12:44                 ` Thomas Monjalon
2021-10-11 13:30                   ` Jerin Jacob
2021-10-19 10:00                     ` Elena Agostini
2021-10-19 18:47                       ` Jerin Jacob
2021-10-19 19:11                         ` Thomas Monjalon
2021-10-19 19:56                           ` [dpdk-dev] [EXT] " Jerin Jacob Kollanukkaran
2021-11-03 19:15 ` [dpdk-dev] [PATCH v4 " eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 1/9] gpudev: introduce GPU device class library eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 2/9] gpudev: add event notification eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 3/9] gpudev: add child device representing a device context eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 4/9] gpudev: support multi-process eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 5/9] gpudev: add memory API eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 6/9] gpudev: add memory barrier eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 7/9] gpudev: add communication flag eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 8/9] gpudev: add communication list eagostini
2021-11-03 19:15   ` [dpdk-dev] [PATCH v4 9/9] doc: add CUDA example in GPU guide eagostini
2021-11-08 18:57 ` [dpdk-dev] [PATCH v5 0/9] GPU library eagostini
2021-11-08 16:25   ` Thomas Monjalon
2021-11-08 18:57   ` [dpdk-dev] [PATCH v5 1/9] gpudev: introduce GPU device class library eagostini
2021-11-08 18:57   ` [dpdk-dev] [PATCH v5 2/9] gpudev: add event notification eagostini
2021-11-08 18:57   ` [dpdk-dev] [PATCH v5 3/9] gpudev: add child device representing a device context eagostini
2021-11-08 18:58   ` [dpdk-dev] [PATCH v5 4/9] gpudev: support multi-process eagostini
2021-11-08 18:58   ` [dpdk-dev] [PATCH v5 5/9] gpudev: add memory API eagostini
2021-11-08 18:58   ` [dpdk-dev] [PATCH v5 6/9] gpudev: add memory barrier eagostini
2021-11-08 18:58   ` [dpdk-dev] [PATCH v5 7/9] gpudev: add communication flag eagostini
2021-11-08 18:58   ` [dpdk-dev] [PATCH v5 8/9] gpudev: add communication list eagostini
2021-11-08 18:58   ` [dpdk-dev] [PATCH v5 9/9] doc: add CUDA example in GPU guide eagostini

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20211009015349.9694-3-eagostini@nvidia.com \
    --to=eagostini@nvidia.com \
    --cc=dev@dpdk.org \
    --cc=thomas@monjalon.net \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).