[dpdk-dev] [PATCH v1 1/2] lib/crypto: add callback handlers for crypto

[dpdk-dev] [PATCH v1 1/2] lib/crypto: add callback handlers for crypto

[Vipin Varghese]

Add callback handlers for enqueue-dequeue operation on crypto
device. The pre-enqueue and post-dequeue are selected on invoke
user registered callback functions.

Use cases:
 - allow user to investigate the contents pre-enqueue.
 - allow user to investigate the contents post-dequeue.
 - modify pre-enqueue and post-dequeue stage content.
 - investigate PMD meta data.

Signed-off-by: Vipin Varghese <vipin.varghese@intel.com>
---
 config/common_base                            |   1 +
 lib/librte_cryptodev/rte_cryptodev.c          | 181 ++++++++++++++++++
 lib/librte_cryptodev/rte_cryptodev.h          | 125 ++++++++++++
 .../rte_cryptodev_version.map                 |   4 +
 4 files changed, 311 insertions(+)

diff --git a/config/common_base b/config/common_base
index 022734f19..dbd5cd8e2 100644
--- a/config/common_base
+++ b/config/common_base
@@ -540,6 +540,7 @@ CONFIG_RTE_LIBRTE_PMD_BBDEV_TURBO_SW=n
 #
 CONFIG_RTE_LIBRTE_CRYPTODEV=y
 CONFIG_RTE_CRYPTO_MAX_DEVS=64
+CONFIG_RTE_CRYPTODEV_ENQDEQ_CALLBACKS=n
 
 #
 # Compile PMD for ARMv8 Crypto device
diff --git a/lib/librte_cryptodev/rte_cryptodev.c b/lib/librte_cryptodev/rte_cryptodev.c
index 00c2cf432..81a844720 100644
--- a/lib/librte_cryptodev/rte_cryptodev.c
+++ b/lib/librte_cryptodev/rte_cryptodev.c
@@ -57,6 +57,11 @@ static struct rte_cryptodev_global cryptodev_globals = {
 /* spinlock for crypto device callbacks */
 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
 
+/* spinlock for pre-enqueue callbacks */
+rte_spinlock_t rte_cryptodev_preenq_cb_lock = RTE_SPINLOCK_INITIALIZER;
+/* spinlock for post-dequeue callbacks */
+rte_spinlock_t rte_cryptodev_pstdeq_cb_lock = RTE_SPINLOCK_INITIALIZER;
+
 
 /**
  * The user application callback description.
@@ -707,6 +712,8 @@ rte_cryptodev_pmd_allocate(const char *name, int socket_id)
 
 		/* init user callbacks */
 		TAILQ_INIT(&(cryptodev->link_intr_cbs));
+		TAILQ_INIT(&(cryptodev->enq_cbs));
+		TAILQ_INIT(&(cryptodev->deq_cbs));
 
 		cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 
@@ -1736,3 +1743,177 @@ rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv,
 
 	return nb_drivers++;
 }
+
+int __rte_experimental
+rte_cryptodev_preenq_callback_register(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg)
+{
+	struct rte_cryptodev *dev;
+	struct rte_cryptodev_enqdeq_callback *user_cb;
+	rte_spinlock_t *lock = &rte_cryptodev_preenq_cb_lock;
+
+	if (!cb_fn)
+		return -EINVAL;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%"PRIu8 " qp_id=%"PRIu8,
+				dev_id, qp_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_crypto_devices[dev_id];
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -EINVAL);
+
+	rte_spinlock_lock(lock);
+
+	TAILQ_FOREACH(user_cb, &(dev->enq_cbs), next) {
+	if ((void *) user_cb->cb_fn == (void *) cb_fn &&
+			user_cb->cb_arg == cb_arg)
+		break;
+	}
+
+	/* create a new callback. */
+	if (user_cb == NULL) {
+		user_cb = rte_zmalloc("CRYPTO_USER_CALLBACK",
+			sizeof(struct rte_cryptodev_enqdeq_callback), 0);
+		if (user_cb != NULL) {
+			user_cb->cb_fn = cb_fn;
+			user_cb->cb_arg = cb_arg;
+			TAILQ_INSERT_TAIL(&(dev->enq_cbs), user_cb, next);
+		}
+	}
+
+	rte_spinlock_unlock(lock);
+
+	return (user_cb == NULL) ? -ENOMEM : 0;
+}
+
+int __rte_experimental
+rte_cryptodev_preenq_callback_unregister(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg)
+{
+	int ret = 0;
+	struct rte_cryptodev *dev;
+	struct rte_cryptodev_enqdeq_callback *cb, *next;
+	rte_spinlock_t *lock = &rte_cryptodev_preenq_cb_lock;
+
+	if (!cb_fn)
+		return -EINVAL;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%"PRIu8 " qp_id=%"PRIu8,
+				dev_id, qp_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_crypto_devices[dev_id];
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -EINVAL);
+
+	rte_spinlock_lock(lock);
+
+	ret = -EINVAL;
+	for (cb = TAILQ_FIRST(&dev->enq_cbs); cb != NULL; cb = next) {
+		next = TAILQ_NEXT(cb, next);
+
+		if (cb->cb_fn != cb_fn || cb->cb_arg != cb_arg)
+			continue;
+
+		if (cb->active == 0) {
+			TAILQ_REMOVE(&(dev->enq_cbs), cb, next);
+			rte_free(cb);
+			ret = 0;
+		} else
+			ret = -EAGAIN;
+	}
+
+	rte_spinlock_unlock(lock);
+
+	return ret;
+}
+
+int __rte_experimental
+rte_cryptodev_pstdeq_callback_register(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg)
+{
+	struct rte_cryptodev *dev;
+	struct rte_cryptodev_enqdeq_callback *user_cb;
+	rte_spinlock_t *lock = &rte_cryptodev_pstdeq_cb_lock;
+
+	if (!cb_fn)
+		return -EINVAL;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%"PRIu8 " qp_id=%"PRIu8,
+				dev_id, qp_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_crypto_devices[dev_id];
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -EINVAL);
+
+	rte_spinlock_lock(lock);
+
+	TAILQ_FOREACH(user_cb, &(dev->deq_cbs), next) {
+		if ((void *) user_cb->cb_fn == (void *) cb_fn &&
+				user_cb->cb_arg == cb_arg)
+			break;
+	}
+
+	/* create a new callback. */
+	if (user_cb == NULL) {
+		user_cb = rte_zmalloc("CRYPTO_USER_CALLBACK",
+			sizeof(struct rte_cryptodev_enqdeq_callback), 0);
+		if (user_cb != NULL) {
+			user_cb->cb_fn = cb_fn;
+			user_cb->cb_arg = cb_arg;
+			TAILQ_INSERT_TAIL(&(dev->deq_cbs), user_cb, next);
+		}
+	}
+
+	rte_spinlock_unlock(lock);
+
+	return (user_cb == NULL) ? -ENOMEM : 0;
+}
+
+int __rte_experimental
+rte_cryptodev_pstdeq_callback_unregister(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg)
+{
+	int ret = 0;
+	struct rte_cryptodev *dev;
+	struct rte_cryptodev_enqdeq_callback *cb, *next;
+	rte_spinlock_t *lock = &rte_cryptodev_pstdeq_cb_lock;
+
+	if (!cb_fn)
+		return -EINVAL;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%"PRIu8 " qp_id=%"PRIu8,
+				dev_id, qp_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_crypto_devices[dev_id];
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -EINVAL);
+
+	rte_spinlock_lock(lock);
+
+	ret = -EINVAL;
+	for (cb = TAILQ_FIRST(&dev->deq_cbs); cb != NULL; cb = next) {
+		next = TAILQ_NEXT(cb, next);
+
+		if (cb->cb_fn != cb_fn || cb->cb_arg != cb_arg)
+			continue;
+
+		if (cb->active == 0) {
+			TAILQ_REMOVE(&(dev->deq_cbs), cb, next);
+			rte_free(cb);
+			ret = 0;
+		} else
+			ret = -EAGAIN;
+	}
+
+	rte_spinlock_unlock(lock);
+
+	return ret;
+}
diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h
index 2d4f6d7e3..629255745 100644
--- a/lib/librte_cryptodev/rte_cryptodev.h
+++ b/lib/librte_cryptodev/rte_cryptodev.h
@@ -18,6 +18,8 @@
 extern "C" {
 #endif
 
+#include <sys/queue.h>
+
 #include "rte_kvargs.h"
 #include "rte_crypto.h"
 #include "rte_dev.h"
@@ -794,9 +796,23 @@ typedef uint16_t (*enqueue_pkt_burst_t)(void *qp,
 
 
 struct rte_cryptodev_callback;
+struct rte_cryptodev_enqdeq_callback;
 
 /** Structure to keep track of registered callbacks */
 TAILQ_HEAD(rte_cryptodev_cb_list, rte_cryptodev_callback);
+TAILQ_HEAD(rte_cryptodev_enq_cb_list, rte_cryptodev_enqdeq_callback);
+TAILQ_HEAD(rte_cryptodev_deq_cb_list, rte_cryptodev_enqdeq_callback);
+
+typedef uint16_t (*rte_cryptodev_enqdeq_cb_fn)(uint8_t dev_id, uint8_t qp_id,
+		struct rte_crypto_op **ops, uint16_t nb_ops,
+		void *cb_arg);
+
+struct rte_cryptodev_enqdeq_callback {
+	TAILQ_ENTRY(rte_cryptodev_enqdeq_callback) next; /* Callbacks list */
+	rte_cryptodev_enqdeq_cb_fn cb_fn; /* Callback address */
+	void *cb_arg; /* Parameter for callback */
+	uint32_t active; /* Callback is executing */
+};
 
 /** The data structure associated with each crypto device. */
 struct rte_cryptodev {
@@ -823,6 +839,9 @@ struct rte_cryptodev {
 	void *security_ctx;
 	/**< Context for security ops */
 
+	struct rte_cryptodev_enq_cb_list enq_cbs;
+	struct rte_cryptodev_deq_cb_list deq_cbs;
+
 	__extension__
 	uint8_t attached : 1;
 	/**< Flag indicating the device is attached */
@@ -907,6 +926,19 @@ rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id,
 	nb_ops = (*dev->dequeue_burst)
 			(dev->data->queue_pairs[qp_id], ops, nb_ops);
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+	struct rte_cryptodev_enqdeq_callback *cb = NULL;
+
+	TAILQ_FOREACH(cb, &(dev->deq_cbs), next) {
+		if (cb->cb_fn == NULL)
+			continue;
+
+		cb->active = 1;
+		nb_ops = cb->cb_fn(dev_id, qp_id, ops, nb_ops, cb->cb_arg);
+		cb->active = 0;
+	}
+#endif
+
 	return nb_ops;
 }
 
@@ -947,6 +979,19 @@ rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id,
 {
 	struct rte_cryptodev *dev = &rte_cryptodevs[dev_id];
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+	struct rte_cryptodev_enqdeq_callback *cb = NULL;
+
+	TAILQ_FOREACH(cb, &(dev->enq_cbs), next) {
+		if (cb->cb_fn == NULL)
+			continue;
+
+		cb->active = 1;
+		nb_ops = cb->cb_fn(dev_id, qp_id, ops, nb_ops, cb->cb_arg);
+		cb->active = 0;
+	}
+#endif
+
 	return (*dev->enqueue_burst)(
 			dev->data->queue_pairs[qp_id], ops, nb_ops);
 }
@@ -1249,6 +1294,86 @@ void * __rte_experimental
 rte_cryptodev_sym_session_get_user_data(
 					struct rte_cryptodev_sym_session *sess);
 
+/**
+ * Register user callback for pre-enqueue of crypto.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param qp_id
+ *   The index of the queue pair.
+ * @param cb_fn
+ *   user callback to register.
+ * @param cb_arg
+ *   user args to be passed during invoke.
+ *
+ * @return
+ *  - On success returns 0.
+ *  - On failure returns < 0.
+ */
+int
+rte_cryptodev_preenq_callback_register(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg);
+
+/**
+ * unregister user callback for pre-enqueue of crypto.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param qp_id
+ *   The index of the queue pair.
+ * @param cb_fn
+ *   user callback to register.
+ * @param cb_arg
+ *   user args to be passed during invoke.
+ *
+ * @return
+ *  - On success returns 0.
+ *  - On failure returns < 0.
+ */
+int
+rte_cryptodev_preenq_callback_unregister(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg);
+
+/**
+ * Register user callback for post-dequeue of crypto.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param qp_id
+ *   The index of the queue pair.
+ * @param cb_fn
+ *   user callback to register.
+ * @param cb_arg
+ *   user args to be passed during invoke.
+ *
+ * @return
+ *  - On success returns 0.
+ *  - On failure returns < 0.
+ */
+int
+rte_cryptodev_pstdeq_callback_register(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg);
+
+/**
+ * unregister user callback for post-dequeue of crypto.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param qp_id
+ *   The index of the queue pair.
+ * @param cb_fn
+ *   user callback to register.
+ * @param cb_arg
+ *   user args to be passed during invoke.
+ *
+ * @return
+ *  - On success returns 0.
+ *  - On failure returns < 0.
+ */
+int
+rte_cryptodev_pstdeq_callback_unregister(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map
index 3deb265ac..e0c1900f3 100644
--- a/lib/librte_cryptodev/rte_cryptodev_version.map
+++ b/lib/librte_cryptodev/rte_cryptodev_version.map
@@ -101,6 +101,10 @@ EXPERIMENTAL {
 	rte_cryptodev_asym_session_init;
 	rte_cryptodev_asym_xform_capability_check_modlen;
 	rte_cryptodev_asym_xform_capability_check_optype;
+	rte_cryptodev_preenq_callback_register;
+	rte_cryptodev_preenq_callback_unregister;
+	rte_cryptodev_pstdeq_callback_register;
+	rte_cryptodev_pstdeq_callback_unregister;
 	rte_cryptodev_sym_get_existing_header_session_size;
 	rte_cryptodev_sym_session_get_user_data;
 	rte_cryptodev_sym_session_pool_create;
-- 
2.17.1

[dpdk-dev] [PATCH v1 2/2] examples/l2fwd-crypto: add callback handlers

[Vipin Varghese]

Register user callback handlers for pre-enqueue and pst-dequeue
for crypto device instance 0 with port 0.

Signed-off-by: Vipin Varghese <vipin.varghese@intel.com>
---
 examples/l2fwd-crypto/main.c | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/examples/l2fwd-crypto/main.c b/examples/l2fwd-crypto/main.c
index e282cb7bf..db2ac2624 100644
--- a/examples/l2fwd-crypto/main.c
+++ b/examples/l2fwd-crypto/main.c
@@ -255,6 +255,20 @@ struct l2fwd_crypto_statistics crypto_statistics[RTE_CRYPTO_MAX_DEVS];
 /* default period is 10 seconds */
 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000;
 
+static uint16_t dump_crypto(__rte_unused uint8_t dev_id,
+		__rte_unused uint8_t qp_id,
+		__rte_unused struct rte_crypto_op **ops,
+		__rte_unused uint16_t nb_ops,
+		__rte_unused void *cb_arg)
+{
+	if (nb_ops)
+		RTE_LOG(DEBUG, L2FWD, " dev_id (%u) qp_id (%u)"
+				" ops (%p) nb_ops (%u)\n",
+				dev_id, qp_id, ops, nb_ops);
+
+	return nb_ops;
+}
+
 /* Print out statistics on packets dropped */
 static void
 print_stats(void)
@@ -2783,6 +2797,18 @@ main(int argc, char **argv)
 				(unsigned)cdev_id);
 	}
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+	ret = rte_cryptodev_preenq_callback_register(0, 0, dump_crypto, NULL);
+	if (ret != 0)
+		RTE_LOG(ERR, L2FWD, " failed to preenq callback register\n");
+	RTE_LOG(INFO, L2FWD, " preenq callback register success\n");
+
+	ret = rte_cryptodev_pstdeq_callback_register(0, 0, dump_crypto, NULL);
+	if (ret != 0)
+		RTE_LOG(ERR, L2FWD, " failed to pstdeq callback register\n");
+	RTE_LOG(INFO, L2FWD, " pstdeq callback register success\n");
+#endif
+
 	/* launch per-lcore init on every lcore */
 	rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options,
 			CALL_MASTER);
@@ -2791,5 +2817,15 @@ main(int argc, char **argv)
 			return -1;
 	}
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+	ret = rte_cryptodev_preenq_callback_unregister(0, 0, dump_crypto, NULL);
+	if (ret != 0)
+		printf(" faield rte_cryptodev_preenq_callback_unregister\n");
+
+	ret = rte_cryptodev_pstdeq_callback_unregister(0, 0, dump_crypto, NULL);
+	if (ret != 0)
+		printf(" failed rte_cryptodev_pstdeq_callback_unregister\n");
+#endif
+
 	return 0;
 }
-- 
2.17.1

[dpdk-dev] [PATCH v2 1/2] lib/crypto: add callback handlers for crypto

[Vipin Varghese]

Add callback handlers for enqueue-dequeue operation on crypto
device. The pre-enqueue and post-dequeue are selected on invoke
user registered callback functions.

Use cases:
 - allow user to investigate the contents pre-enqueue.
 - allow user to investigate the contents post-dequeue.
 - modify pre-enqueue and post-dequeue stage content.
 - investigate PMD meta data.

Signed-off-by: Vipin Varghese <vipin.varghese@intel.com>
---
 config/common_base                            |   1 +
 lib/librte_cryptodev/rte_cryptodev.c          | 181 ++++++++++++++++++
 lib/librte_cryptodev/rte_cryptodev.h          | 125 ++++++++++++
 .../rte_cryptodev_version.map                 |   4 +
 4 files changed, 311 insertions(+)

diff --git a/config/common_base b/config/common_base
index 022734f19..dbd5cd8e2 100644
--- a/config/common_base
+++ b/config/common_base
@@ -540,6 +540,7 @@ CONFIG_RTE_LIBRTE_PMD_BBDEV_TURBO_SW=n
 #
 CONFIG_RTE_LIBRTE_CRYPTODEV=y
 CONFIG_RTE_CRYPTO_MAX_DEVS=64
+CONFIG_RTE_CRYPTODEV_ENQDEQ_CALLBACKS=n
 
 #
 # Compile PMD for ARMv8 Crypto device
diff --git a/lib/librte_cryptodev/rte_cryptodev.c b/lib/librte_cryptodev/rte_cryptodev.c
index 00c2cf432..81a844720 100644
--- a/lib/librte_cryptodev/rte_cryptodev.c
+++ b/lib/librte_cryptodev/rte_cryptodev.c
@@ -57,6 +57,11 @@ static struct rte_cryptodev_global cryptodev_globals = {
 /* spinlock for crypto device callbacks */
 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
 
+/* spinlock for pre-enqueue callbacks */
+rte_spinlock_t rte_cryptodev_preenq_cb_lock = RTE_SPINLOCK_INITIALIZER;
+/* spinlock for post-dequeue callbacks */
+rte_spinlock_t rte_cryptodev_pstdeq_cb_lock = RTE_SPINLOCK_INITIALIZER;
+
 
 /**
  * The user application callback description.
@@ -707,6 +712,8 @@ rte_cryptodev_pmd_allocate(const char *name, int socket_id)
 
 		/* init user callbacks */
 		TAILQ_INIT(&(cryptodev->link_intr_cbs));
+		TAILQ_INIT(&(cryptodev->enq_cbs));
+		TAILQ_INIT(&(cryptodev->deq_cbs));
 
 		cryptodev->attached = RTE_CRYPTODEV_ATTACHED;
 
@@ -1736,3 +1743,177 @@ rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv,
 
 	return nb_drivers++;
 }
+
+int __rte_experimental
+rte_cryptodev_preenq_callback_register(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg)
+{
+	struct rte_cryptodev *dev;
+	struct rte_cryptodev_enqdeq_callback *user_cb;
+	rte_spinlock_t *lock = &rte_cryptodev_preenq_cb_lock;
+
+	if (!cb_fn)
+		return -EINVAL;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%"PRIu8 " qp_id=%"PRIu8,
+				dev_id, qp_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_crypto_devices[dev_id];
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -EINVAL);
+
+	rte_spinlock_lock(lock);
+
+	TAILQ_FOREACH(user_cb, &(dev->enq_cbs), next) {
+	if ((void *) user_cb->cb_fn == (void *) cb_fn &&
+			user_cb->cb_arg == cb_arg)
+		break;
+	}
+
+	/* create a new callback. */
+	if (user_cb == NULL) {
+		user_cb = rte_zmalloc("CRYPTO_USER_CALLBACK",
+			sizeof(struct rte_cryptodev_enqdeq_callback), 0);
+		if (user_cb != NULL) {
+			user_cb->cb_fn = cb_fn;
+			user_cb->cb_arg = cb_arg;
+			TAILQ_INSERT_TAIL(&(dev->enq_cbs), user_cb, next);
+		}
+	}
+
+	rte_spinlock_unlock(lock);
+
+	return (user_cb == NULL) ? -ENOMEM : 0;
+}
+
+int __rte_experimental
+rte_cryptodev_preenq_callback_unregister(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg)
+{
+	int ret = 0;
+	struct rte_cryptodev *dev;
+	struct rte_cryptodev_enqdeq_callback *cb, *next;
+	rte_spinlock_t *lock = &rte_cryptodev_preenq_cb_lock;
+
+	if (!cb_fn)
+		return -EINVAL;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%"PRIu8 " qp_id=%"PRIu8,
+				dev_id, qp_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_crypto_devices[dev_id];
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -EINVAL);
+
+	rte_spinlock_lock(lock);
+
+	ret = -EINVAL;
+	for (cb = TAILQ_FIRST(&dev->enq_cbs); cb != NULL; cb = next) {
+		next = TAILQ_NEXT(cb, next);
+
+		if (cb->cb_fn != cb_fn || cb->cb_arg != cb_arg)
+			continue;
+
+		if (cb->active == 0) {
+			TAILQ_REMOVE(&(dev->enq_cbs), cb, next);
+			rte_free(cb);
+			ret = 0;
+		} else
+			ret = -EAGAIN;
+	}
+
+	rte_spinlock_unlock(lock);
+
+	return ret;
+}
+
+int __rte_experimental
+rte_cryptodev_pstdeq_callback_register(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg)
+{
+	struct rte_cryptodev *dev;
+	struct rte_cryptodev_enqdeq_callback *user_cb;
+	rte_spinlock_t *lock = &rte_cryptodev_pstdeq_cb_lock;
+
+	if (!cb_fn)
+		return -EINVAL;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%"PRIu8 " qp_id=%"PRIu8,
+				dev_id, qp_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_crypto_devices[dev_id];
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -EINVAL);
+
+	rte_spinlock_lock(lock);
+
+	TAILQ_FOREACH(user_cb, &(dev->deq_cbs), next) {
+		if ((void *) user_cb->cb_fn == (void *) cb_fn &&
+				user_cb->cb_arg == cb_arg)
+			break;
+	}
+
+	/* create a new callback. */
+	if (user_cb == NULL) {
+		user_cb = rte_zmalloc("CRYPTO_USER_CALLBACK",
+			sizeof(struct rte_cryptodev_enqdeq_callback), 0);
+		if (user_cb != NULL) {
+			user_cb->cb_fn = cb_fn;
+			user_cb->cb_arg = cb_arg;
+			TAILQ_INSERT_TAIL(&(dev->deq_cbs), user_cb, next);
+		}
+	}
+
+	rte_spinlock_unlock(lock);
+
+	return (user_cb == NULL) ? -ENOMEM : 0;
+}
+
+int __rte_experimental
+rte_cryptodev_pstdeq_callback_unregister(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg)
+{
+	int ret = 0;
+	struct rte_cryptodev *dev;
+	struct rte_cryptodev_enqdeq_callback *cb, *next;
+	rte_spinlock_t *lock = &rte_cryptodev_pstdeq_cb_lock;
+
+	if (!cb_fn)
+		return -EINVAL;
+
+	if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%"PRIu8 " qp_id=%"PRIu8,
+				dev_id, qp_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_crypto_devices[dev_id];
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -EINVAL);
+
+	rte_spinlock_lock(lock);
+
+	ret = -EINVAL;
+	for (cb = TAILQ_FIRST(&dev->deq_cbs); cb != NULL; cb = next) {
+		next = TAILQ_NEXT(cb, next);
+
+		if (cb->cb_fn != cb_fn || cb->cb_arg != cb_arg)
+			continue;
+
+		if (cb->active == 0) {
+			TAILQ_REMOVE(&(dev->deq_cbs), cb, next);
+			rte_free(cb);
+			ret = 0;
+		} else
+			ret = -EAGAIN;
+	}
+
+	rte_spinlock_unlock(lock);
+
+	return ret;
+}
diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h
index 2d4f6d7e3..629255745 100644
--- a/lib/librte_cryptodev/rte_cryptodev.h
+++ b/lib/librte_cryptodev/rte_cryptodev.h
@@ -18,6 +18,8 @@
 extern "C" {
 #endif
 
+#include <sys/queue.h>
+
 #include "rte_kvargs.h"
 #include "rte_crypto.h"
 #include "rte_dev.h"
@@ -794,9 +796,23 @@ typedef uint16_t (*enqueue_pkt_burst_t)(void *qp,
 
 
 struct rte_cryptodev_callback;
+struct rte_cryptodev_enqdeq_callback;
 
 /** Structure to keep track of registered callbacks */
 TAILQ_HEAD(rte_cryptodev_cb_list, rte_cryptodev_callback);
+TAILQ_HEAD(rte_cryptodev_enq_cb_list, rte_cryptodev_enqdeq_callback);
+TAILQ_HEAD(rte_cryptodev_deq_cb_list, rte_cryptodev_enqdeq_callback);
+
+typedef uint16_t (*rte_cryptodev_enqdeq_cb_fn)(uint8_t dev_id, uint8_t qp_id,
+		struct rte_crypto_op **ops, uint16_t nb_ops,
+		void *cb_arg);
+
+struct rte_cryptodev_enqdeq_callback {
+	TAILQ_ENTRY(rte_cryptodev_enqdeq_callback) next; /* Callbacks list */
+	rte_cryptodev_enqdeq_cb_fn cb_fn; /* Callback address */
+	void *cb_arg; /* Parameter for callback */
+	uint32_t active; /* Callback is executing */
+};
 
 /** The data structure associated with each crypto device. */
 struct rte_cryptodev {
@@ -823,6 +839,9 @@ struct rte_cryptodev {
 	void *security_ctx;
 	/**< Context for security ops */
 
+	struct rte_cryptodev_enq_cb_list enq_cbs;
+	struct rte_cryptodev_deq_cb_list deq_cbs;
+
 	__extension__
 	uint8_t attached : 1;
 	/**< Flag indicating the device is attached */
@@ -907,6 +926,19 @@ rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id,
 	nb_ops = (*dev->dequeue_burst)
 			(dev->data->queue_pairs[qp_id], ops, nb_ops);
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+	struct rte_cryptodev_enqdeq_callback *cb = NULL;
+
+	TAILQ_FOREACH(cb, &(dev->deq_cbs), next) {
+		if (cb->cb_fn == NULL)
+			continue;
+
+		cb->active = 1;
+		nb_ops = cb->cb_fn(dev_id, qp_id, ops, nb_ops, cb->cb_arg);
+		cb->active = 0;
+	}
+#endif
+
 	return nb_ops;
 }
 
@@ -947,6 +979,19 @@ rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id,
 {
 	struct rte_cryptodev *dev = &rte_cryptodevs[dev_id];
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+	struct rte_cryptodev_enqdeq_callback *cb = NULL;
+
+	TAILQ_FOREACH(cb, &(dev->enq_cbs), next) {
+		if (cb->cb_fn == NULL)
+			continue;
+
+		cb->active = 1;
+		nb_ops = cb->cb_fn(dev_id, qp_id, ops, nb_ops, cb->cb_arg);
+		cb->active = 0;
+	}
+#endif
+
 	return (*dev->enqueue_burst)(
 			dev->data->queue_pairs[qp_id], ops, nb_ops);
 }
@@ -1249,6 +1294,86 @@ void * __rte_experimental
 rte_cryptodev_sym_session_get_user_data(
 					struct rte_cryptodev_sym_session *sess);
 
+/**
+ * Register user callback for pre-enqueue of crypto.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param qp_id
+ *   The index of the queue pair.
+ * @param cb_fn
+ *   user callback to register.
+ * @param cb_arg
+ *   user args to be passed during invoke.
+ *
+ * @return
+ *  - On success returns 0.
+ *  - On failure returns < 0.
+ */
+int
+rte_cryptodev_preenq_callback_register(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg);
+
+/**
+ * unregister user callback for pre-enqueue of crypto.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param qp_id
+ *   The index of the queue pair.
+ * @param cb_fn
+ *   user callback to register.
+ * @param cb_arg
+ *   user args to be passed during invoke.
+ *
+ * @return
+ *  - On success returns 0.
+ *  - On failure returns < 0.
+ */
+int
+rte_cryptodev_preenq_callback_unregister(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg);
+
+/**
+ * Register user callback for post-dequeue of crypto.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param qp_id
+ *   The index of the queue pair.
+ * @param cb_fn
+ *   user callback to register.
+ * @param cb_arg
+ *   user args to be passed during invoke.
+ *
+ * @return
+ *  - On success returns 0.
+ *  - On failure returns < 0.
+ */
+int
+rte_cryptodev_pstdeq_callback_register(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg);
+
+/**
+ * unregister user callback for post-dequeue of crypto.
+ *
+ * @param dev_id
+ *   The identifier of the device.
+ * @param qp_id
+ *   The index of the queue pair.
+ * @param cb_fn
+ *   user callback to register.
+ * @param cb_arg
+ *   user args to be passed during invoke.
+ *
+ * @return
+ *  - On success returns 0.
+ *  - On failure returns < 0.
+ */
+int
+rte_cryptodev_pstdeq_callback_unregister(uint8_t dev_id, uint8_t qp_id,
+		rte_cryptodev_enqdeq_cb_fn cb_fn, void *cb_arg);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map
index 3deb265ac..e0c1900f3 100644
--- a/lib/librte_cryptodev/rte_cryptodev_version.map
+++ b/lib/librte_cryptodev/rte_cryptodev_version.map
@@ -101,6 +101,10 @@ EXPERIMENTAL {
 	rte_cryptodev_asym_session_init;
 	rte_cryptodev_asym_xform_capability_check_modlen;
 	rte_cryptodev_asym_xform_capability_check_optype;
+	rte_cryptodev_preenq_callback_register;
+	rte_cryptodev_preenq_callback_unregister;
+	rte_cryptodev_pstdeq_callback_register;
+	rte_cryptodev_pstdeq_callback_unregister;
 	rte_cryptodev_sym_get_existing_header_session_size;
 	rte_cryptodev_sym_session_get_user_data;
 	rte_cryptodev_sym_session_pool_create;
-- 
2.17.1

[dpdk-dev] [PATCH v2 2/2] examples/l2fwd-crypto: add callback handlers

[Vipin Varghese]

Register user callback handlers for pre-enqueue and pst-dequeue
for crypto device instance 0 with port 0.

Signed-off-by: Vipin Varghese <vipin.varghese@intel.com>
---
 examples/l2fwd-crypto/main.c | 38 ++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/examples/l2fwd-crypto/main.c b/examples/l2fwd-crypto/main.c
index e282cb7bf..c42b352f8 100644
--- a/examples/l2fwd-crypto/main.c
+++ b/examples/l2fwd-crypto/main.c
@@ -255,6 +255,22 @@ struct l2fwd_crypto_statistics crypto_statistics[RTE_CRYPTO_MAX_DEVS];
 /* default period is 10 seconds */
 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000;
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+static uint16_t dump_crypto(__rte_unused uint8_t dev_id,
+		__rte_unused uint8_t qp_id,
+		__rte_unused struct rte_crypto_op **ops,
+		__rte_unused uint16_t nb_ops,
+		__rte_unused void *cb_arg)
+{
+	if (nb_ops)
+		RTE_LOG(DEBUG, L2FWD, " dev_id (%u) qp_id (%u)"
+				" ops (%p) nb_ops (%u)\n",
+				dev_id, qp_id, ops, nb_ops);
+
+	return nb_ops;
+}
+#endif
+
 /* Print out statistics on packets dropped */
 static void
 print_stats(void)
@@ -2783,6 +2799,18 @@ main(int argc, char **argv)
 				(unsigned)cdev_id);
 	}
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+	ret = rte_cryptodev_preenq_callback_register(0, 0, dump_crypto, NULL);
+	if (ret != 0)
+		RTE_LOG(ERR, L2FWD, " failed to preenq callback register\n");
+	RTE_LOG(INFO, L2FWD, " preenq callback register success\n");
+
+	ret = rte_cryptodev_pstdeq_callback_register(0, 0, dump_crypto, NULL);
+	if (ret != 0)
+		RTE_LOG(ERR, L2FWD, " failed to pstdeq callback register\n");
+	RTE_LOG(INFO, L2FWD, " pstdeq callback register success\n");
+#endif
+
 	/* launch per-lcore init on every lcore */
 	rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options,
 			CALL_MASTER);
@@ -2791,5 +2819,15 @@ main(int argc, char **argv)
 			return -1;
 	}
 
+#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
+	ret = rte_cryptodev_preenq_callback_unregister(0, 0, dump_crypto, NULL);
+	if (ret != 0)
+		printf(" faield rte_cryptodev_preenq_callback_unregister\n");
+
+	ret = rte_cryptodev_pstdeq_callback_unregister(0, 0, dump_crypto, NULL);
+	if (ret != 0)
+		printf(" failed rte_cryptodev_pstdeq_callback_unregister\n");
+#endif
+
 	return 0;
 }
-- 
2.17.1

Re: [dpdk-dev] [PATCH v2 2/2] examples/l2fwd-crypto: add callback handlers

[Ruifeng Wang (Arm Technology China)]

Hi,

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Vipin Varghese
> Sent: Monday, June 10, 2019 14:30
> To: keith.wiles@intel.com; dev@dpdk.org; pablo.de.lara.guarch@intel.com;
> Akhil.goyal@nxp.com; declan.doherty@intel.com
> Cc: sanjay.padubidri@intel.com; Vipin Varghese <vipin.varghese@intel.com>
> Subject: [dpdk-dev] [PATCH v2 2/2] examples/l2fwd-crypto: add callback
> handlers
> 
> Register user callback handlers for pre-enqueue and pst-dequeue for crypto
> device instance 0 with port 0.
> 
> Signed-off-by: Vipin Varghese <vipin.varghese@intel.com>
> ---
>  examples/l2fwd-crypto/main.c | 38
> ++++++++++++++++++++++++++++++++++++
>  1 file changed, 38 insertions(+)
> 
> diff --git a/examples/l2fwd-crypto/main.c b/examples/l2fwd-crypto/main.c
> index e282cb7bf..c42b352f8 100644
> --- a/examples/l2fwd-crypto/main.c
> +++ b/examples/l2fwd-crypto/main.c
> @@ -255,6 +255,22 @@ struct l2fwd_crypto_statistics
> crypto_statistics[RTE_CRYPTO_MAX_DEVS];
>  /* default period is 10 seconds */
>  static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000;
> 
> +#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
> +static uint16_t dump_crypto(__rte_unused uint8_t dev_id,
> +		__rte_unused uint8_t qp_id,
> +		__rte_unused struct rte_crypto_op **ops,
> +		__rte_unused uint16_t nb_ops,
> +		__rte_unused void *cb_arg)
> +{
> +	if (nb_ops)
> +		RTE_LOG(DEBUG, L2FWD, " dev_id (%u) qp_id (%u)"
> +				" ops (%p) nb_ops (%u)\n",
> +				dev_id, qp_id, ops, nb_ops);
> +
> +	return nb_ops;
> +}
> +#endif
> +
>  /* Print out statistics on packets dropped */  static void
>  print_stats(void)
> @@ -2783,6 +2799,18 @@ main(int argc, char **argv)
>  				(unsigned)cdev_id);
>  	}
> 
> +#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
> +	ret = rte_cryptodev_preenq_callback_register(0, 0, dump_crypto,
> NULL);
> +	if (ret != 0)
> +		RTE_LOG(ERR, L2FWD, " failed to preenq callback register\n");
> +	RTE_LOG(INFO, L2FWD, " preenq callback register success\n");
> +
> +	ret = rte_cryptodev_pstdeq_callback_register(0, 0, dump_crypto,
> NULL);
> +	if (ret != 0)
> +		RTE_LOG(ERR, L2FWD, " failed to pstdeq callback register\n");
> +	RTE_LOG(INFO, L2FWD, " pstdeq callback register success\n"); #endif
> +
>  	/* launch per-lcore init on every lcore */
>  	rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void
> *)&options,
>  			CALL_MASTER);
> @@ -2791,5 +2819,15 @@ main(int argc, char **argv)
>  			return -1;
>  	}
> 
> +#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
> +	ret = rte_cryptodev_preenq_callback_unregister(0, 0, dump_crypto,
> NULL);
> +	if (ret != 0)
> +		printf(" faield

Observed typo here 'faield' -> 'failed'  ;)

> rte_cryptodev_preenq_callback_unregister\n");
> +
> +	ret = rte_cryptodev_pstdeq_callback_unregister(0, 0, dump_crypto,
> NULL);
> +	if (ret != 0)
> +		printf(" failed
> rte_cryptodev_pstdeq_callback_unregister\n");
> +#endif
> +
>  	return 0;
>  }
> --
> 2.17.1

Re: [dpdk-dev] [PATCH v2 1/2] lib/crypto: add callback handlers for crypto

[Akhil Goyal]

Hi Vipin,

> 
> Add callback handlers for enqueue-dequeue operation on crypto
> device. The pre-enqueue and post-dequeue are selected on invoke
> user registered callback functions.
> 
> Use cases:
>  - allow user to investigate the contents pre-enqueue.
>  - allow user to investigate the contents post-dequeue.
>  - modify pre-enqueue and post-dequeue stage content.
>  - investigate PMD meta data.
> 
> Signed-off-by: Vipin Varghese <vipin.varghese@intel.com>
> ---

[..]

> @@ -907,6 +926,19 @@ rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t
> qp_id,
>  	nb_ops = (*dev->dequeue_burst)
>  			(dev->data->queue_pairs[qp_id], ops, nb_ops);
> 
> +#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
> +	struct rte_cryptodev_enqdeq_callback *cb = NULL;
> +
> +	TAILQ_FOREACH(cb, &(dev->deq_cbs), next) {
> +		if (cb->cb_fn == NULL)
> +			continue;
> +
> +		cb->active = 1;
> +		nb_ops = cb->cb_fn(dev_id, qp_id, ops, nb_ops, cb->cb_arg);
> +		cb->active = 0;
> +	}

Shouldn't this cb->active be set atomically.

Will it be thread safe? There may be multiple threads enqueuing on the same device.
One may be executing while the other finished and set the active to 0, and may unregister
While the some other thread is still executing.

One more thing, it would be better to have a debug prints about how many nb_ops have been
Successfully passed through each of the callback.
And in the callback, it should be assumed that it will return back just after the first failed ops, so that
The application can free the remaining one. This should be documented in the callback API.

> +#endif
> +
>  	return nb_ops;
>  }
> 

Re: [dpdk-dev] [PATCH v2 1/2] lib/crypto: add callback handlers for crypto

[Varghese, Vipin]

Thanks Akhil, I will work on the suggested inputs.

> -----Original Message-----
> From: Akhil Goyal <akhil.goyal@nxp.com>
> Sent: Thursday, June 27, 2019 7:55 PM
> To: Varghese, Vipin <vipin.varghese@intel.com>; Wiles, Keith
> <keith.wiles@intel.com>; dev@dpdk.org; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>; Doherty, Declan
> <declan.doherty@intel.com>
> Cc: Padubidri, Sanjay A <sanjay.padubidri@intel.com>
> Subject: RE: [PATCH v2 1/2] lib/crypto: add callback handlers for crypto
> 
> Hi Vipin,
> 
> >
> > Add callback handlers for enqueue-dequeue operation on crypto device.
> > The pre-enqueue and post-dequeue are selected on invoke user
> > registered callback functions.
> >
> > Use cases:
> >  - allow user to investigate the contents pre-enqueue.
> >  - allow user to investigate the contents post-dequeue.
> >  - modify pre-enqueue and post-dequeue stage content.
> >  - investigate PMD meta data.
> >
> > Signed-off-by: Vipin Varghese <vipin.varghese@intel.com>
> > ---
> 
> [..]
> 
> > @@ -907,6 +926,19 @@ rte_cryptodev_dequeue_burst(uint8_t dev_id,
> > uint16_t qp_id,
> >  	nb_ops = (*dev->dequeue_burst)
> >  			(dev->data->queue_pairs[qp_id], ops, nb_ops);
> >
> > +#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
> > +	struct rte_cryptodev_enqdeq_callback *cb = NULL;
> > +
> > +	TAILQ_FOREACH(cb, &(dev->deq_cbs), next) {
> > +		if (cb->cb_fn == NULL)
> > +			continue;
> > +
> > +		cb->active = 1;
> > +		nb_ops = cb->cb_fn(dev_id, qp_id, ops, nb_ops, cb->cb_arg);
> > +		cb->active = 0;
> > +	}
> 
> Shouldn't this cb->active be set atomically.
> 
> Will it be thread safe? There may be multiple threads enqueuing on the same
> device.
> One may be executing while the other finished and set the active to 0, and may
> unregister While the some other thread is still executing.
> 
> One more thing, it would be better to have a debug prints about how many
> nb_ops have been Successfully passed through each of the callback.
> And in the callback, it should be assumed that it will return back just after the first
> failed ops, so that The application can free the remaining one. This should be
> documented in the callback API.
> 
> > +#endif
> > +
> >  	return nb_ops;
> >  }
> >

Re: [dpdk-dev] [PATCH v2 2/2] examples/l2fwd-crypto: add callback handlers

[Varghese, Vipin]

Thanks Ruifeng, will work on the suggested changes.

> -----Original Message-----
> From: Ruifeng Wang (Arm Technology China) <Ruifeng.Wang@arm.com>
> Sent: Friday, June 21, 2019 12:37 PM
> To: Varghese, Vipin <vipin.varghese@intel.com>; Wiles, Keith
> <keith.wiles@intel.com>; dev@dpdk.org; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>; Akhil.goyal@nxp.com; Doherty, Declan
> <declan.doherty@intel.com>
> Cc: Padubidri, Sanjay A <sanjay.padubidri@intel.com>; nd <nd@arm.com>
> Subject: RE: [dpdk-dev] [PATCH v2 2/2] examples/l2fwd-crypto: add callback
> handlers
> 
> Hi,
> 
> > -----Original Message-----
> > From: dev <dev-bounces@dpdk.org> On Behalf Of Vipin Varghese
> > Sent: Monday, June 10, 2019 14:30
> > To: keith.wiles@intel.com; dev@dpdk.org;
> > pablo.de.lara.guarch@intel.com; Akhil.goyal@nxp.com;
> > declan.doherty@intel.com
> > Cc: sanjay.padubidri@intel.com; Vipin Varghese
> > <vipin.varghese@intel.com>
> > Subject: [dpdk-dev] [PATCH v2 2/2] examples/l2fwd-crypto: add callback
> > handlers
> >
> > Register user callback handlers for pre-enqueue and pst-dequeue for
> > crypto device instance 0 with port 0.
> >
> > Signed-off-by: Vipin Varghese <vipin.varghese@intel.com>
> > ---
> >  examples/l2fwd-crypto/main.c | 38
> > ++++++++++++++++++++++++++++++++++++
> >  1 file changed, 38 insertions(+)
> >
> > diff --git a/examples/l2fwd-crypto/main.c
> > b/examples/l2fwd-crypto/main.c index e282cb7bf..c42b352f8 100644
> > --- a/examples/l2fwd-crypto/main.c
> > +++ b/examples/l2fwd-crypto/main.c
> > @@ -255,6 +255,22 @@ struct l2fwd_crypto_statistics
> > crypto_statistics[RTE_CRYPTO_MAX_DEVS];
> >  /* default period is 10 seconds */
> >  static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000;
> >
> > +#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS static uint16_t
> > +dump_crypto(__rte_unused uint8_t dev_id,
> > +		__rte_unused uint8_t qp_id,
> > +		__rte_unused struct rte_crypto_op **ops,
> > +		__rte_unused uint16_t nb_ops,
> > +		__rte_unused void *cb_arg)
> > +{
> > +	if (nb_ops)
> > +		RTE_LOG(DEBUG, L2FWD, " dev_id (%u) qp_id (%u)"
> > +				" ops (%p) nb_ops (%u)\n",
> > +				dev_id, qp_id, ops, nb_ops);
> > +
> > +	return nb_ops;
> > +}
> > +#endif
> > +
> >  /* Print out statistics on packets dropped */  static void
> >  print_stats(void)
> > @@ -2783,6 +2799,18 @@ main(int argc, char **argv)
> >  				(unsigned)cdev_id);
> >  	}
> >
> > +#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
> > +	ret = rte_cryptodev_preenq_callback_register(0, 0, dump_crypto,
> > NULL);
> > +	if (ret != 0)
> > +		RTE_LOG(ERR, L2FWD, " failed to preenq callback register\n");
> > +	RTE_LOG(INFO, L2FWD, " preenq callback register success\n");
> > +
> > +	ret = rte_cryptodev_pstdeq_callback_register(0, 0, dump_crypto,
> > NULL);
> > +	if (ret != 0)
> > +		RTE_LOG(ERR, L2FWD, " failed to pstdeq callback register\n");
> > +	RTE_LOG(INFO, L2FWD, " pstdeq callback register success\n"); #endif
> > +
> >  	/* launch per-lcore init on every lcore */
> >  	rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options,
> >  			CALL_MASTER);
> > @@ -2791,5 +2819,15 @@ main(int argc, char **argv)
> >  			return -1;
> >  	}
> >
> > +#ifdef RTE_CRYPTODEV_ENQDEQ_CALLBACKS
> > +	ret = rte_cryptodev_preenq_callback_unregister(0, 0, dump_crypto,
> > NULL);
> > +	if (ret != 0)
> > +		printf(" faield
> 
> Observed typo here 'faield' -> 'failed'  ;)
> 
> > rte_cryptodev_preenq_callback_unregister\n");
> > +
> > +	ret = rte_cryptodev_pstdeq_callback_unregister(0, 0, dump_crypto,
> > NULL);
> > +	if (ret != 0)
> > +		printf(" failed
> > rte_cryptodev_pstdeq_callback_unregister\n");
> > +#endif
> > +
> >  	return 0;
> >  }
> > --
> > 2.17.1