[dpdk-dev] [dpdk-dev v4 1/4] cryptodev: add symmetric crypto data-path APIs

[Fan Zhang <roy.fan.zhang@intel.com>]

This patch adds data-path APIs to cryptodev. The APIs are organized as
a data structure containing function pointers for different enqueue and
dequeue operations. An added public API is added to obtain the function
pointers and necessary queue pair data from the device queue pair.

This patch depends on patch-72157 ("cryptodev: add function to check
if qp was setup")

Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
Signed-off-by: Piotr Bronowski <piotrx.bronowski@intel.com>
---
 lib/librte_cryptodev/rte_crypto_sym.h         |  48 +++++
 lib/librte_cryptodev/rte_cryptodev.c          |  22 +++
 lib/librte_cryptodev/rte_cryptodev.h          | 173 +++++++++++++++++-
 lib/librte_cryptodev/rte_cryptodev_pmd.h      |  12 +-
 .../rte_cryptodev_version.map                 |   4 +
 5 files changed, 255 insertions(+), 4 deletions(-)

diff --git a/lib/librte_cryptodev/rte_crypto_sym.h b/lib/librte_cryptodev/rte_crypto_sym.h
index da961a19d..e237e3cfa 100644
--- a/lib/librte_cryptodev/rte_crypto_sym.h
+++ b/lib/librte_cryptodev/rte_crypto_sym.h
@@ -87,6 +87,54 @@ union rte_crypto_sym_ofs {
 	} ofs;
 };
 
+
+/**
+ * Asynchronous operation job descriptor.
+ * Used by HW crypto devices direct API call that supports such activity
+ **/
+struct rte_crypto_sym_job {
+	union {
+		/**
+		 * When RTE_CRYPTO_HW_ENQ_FLAG_IS_SGL bit is set in flags, sgl
+		 * field is used as input data. Otherwise data_iova is
+		 * used.
+		 **/
+		rte_iova_t data_iova;
+		struct rte_crypto_sgl *sgl;
+	};
+	union {
+		/**
+		 * Different than cryptodev ops, all ofs and len fields have
+		 * the unit of bytes (including Snow3G/Kasumi/Zuc.
+		 **/
+		struct {
+			uint32_t cipher_ofs;
+			uint32_t cipher_len;
+		} cipher_only;
+		struct {
+			uint32_t auth_ofs;
+			uint32_t auth_len;
+			rte_iova_t digest_iova;
+		} auth_only;
+		struct {
+			uint32_t aead_ofs;
+			uint32_t aead_len;
+			rte_iova_t tag_iova;
+			uint8_t *aad;
+			rte_iova_t aad_iova;
+		} aead;
+		struct {
+			uint32_t cipher_ofs;
+			uint32_t cipher_len;
+			uint32_t auth_ofs;
+			uint32_t auth_len;
+			rte_iova_t digest_iova;
+		} chain;
+	};
+	uint8_t *iv;
+	rte_iova_t iv_iova;
+};
+
 /** Symmetric Cipher Algorithms */
 enum rte_crypto_cipher_algorithm {
 	RTE_CRYPTO_CIPHER_NULL = 1,
diff --git a/lib/librte_cryptodev/rte_cryptodev.c b/lib/librte_cryptodev/rte_cryptodev.c
index 705387b8b..5d5f84e27 100644
--- a/lib/librte_cryptodev/rte_cryptodev.c
+++ b/lib/librte_cryptodev/rte_cryptodev.c
@@ -1866,6 +1866,28 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
 	return dev->dev_ops->sym_cpu_process(dev, sess, ofs, vec);
 }
 
+int
+rte_cryptodev_sym_get_hw_ops(uint8_t dev_id, uint16_t qp_id,
+		struct rte_crypto_hw_ops *hw_ops)
+{
+	struct rte_cryptodev *dev;
+
+	if (!hw_ops)
+		return -EINVAL;
+
+	memset(hw_ops, 0, sizeof(*hw_ops));
+
+	if (!rte_cryptodev_get_qp_status(dev_id, qp_id))
+		return -EINVAL;
+
+	dev = rte_cryptodev_pmd_get_dev(dev_id);
+	if (!(dev->feature_flags & RTE_CRYPTODEV_FF_SYM_HW_DIRECT_API) ||
+			*dev->dev_ops->sym_get_hw_ops == NULL)
+		return -ENOTSUP;
+
+	return dev->dev_ops->sym_get_hw_ops(dev, qp_id, hw_ops);
+}
+
 /** Initialise rte_crypto_op mempool element */
 static void
 rte_crypto_op_init(struct rte_mempool *mempool,
diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h
index d01a65825..7cd2095d7 100644
--- a/lib/librte_cryptodev/rte_cryptodev.h
+++ b/lib/librte_cryptodev/rte_cryptodev.h
@@ -466,7 +466,8 @@ rte_cryptodev_asym_get_xform_enum(enum rte_crypto_asym_xform_type *xform_enum,
 /**< Support symmetric session-less operations */
 #define RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA		(1ULL << 23)
 /**< Support operations on data which is not byte aligned */
-
+#define RTE_CRYPTODEV_FF_SYM_HW_DIRECT_API		(1ULL << 24)
+/**< Support hardware accelerator specific raw data as input */
 
 /**
  * Get the name of a crypto device feature flag
@@ -737,7 +738,7 @@ rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
  *   - 1: qp was configured
  *   - -ENODEV: device was not configured
  */
-int
+__rte_experimental int
 rte_cryptodev_get_qp_status(uint8_t dev_id, uint16_t queue_pair_id);
 
 /**
@@ -1348,6 +1349,174 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
 	struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs,
 	struct rte_crypto_sym_vec *vec);
 
+
+/* HW direct symmetric crypto data-path APIs */
+
+/* Bit-masks used for enqueuing job */
+#define RTE_CRYPTO_HW_ENQ_FLAG_START		(1ULL << 0)
+/**< Bit-mask to indicate the first job in a burst. With this bit set the
+ *   driver may write but not read the drv_data buffer, otherwise the driver
+ *   shall read and update the drv_data.
+ */
+#define RTE_CRYPTO_HW_ENQ_FLAG_SET_OPAQUE	(1ULL << 1)
+/**< Bit-mask to indicate write opaque pointer into HW crypto descriptor. */
+#define RTE_CRYPTO_HW_ENQ_FLAG_END		(1ULL << 2)
+/**< Bit-mask to indicate the last job in a burst. With this bit set the
+ *   driver may read but not write the drv_data buffer, and kick the HW to
+ *   start processing all jobs written.
+ */
+#define RTE_CRYPTO_HW_ENQ_FLAG_IS_SGL		(1ULL << 3)
+/**< Bit-mask to indicate the input job is an SGL buffer */
+
+/* Bit-masks used for dequeuing job */
+#define RTE_CRYPTO_HW_DEQ_FLAG_START		(1ULL << 0)
+/**< Bit-mask to indicate the first job to be dequeued. With this bit set the
+ *   driver may write but not read the drv_data buffer, otherwise the driver
+ *   shall read and update the drv_data.
+ */
+#define RTE_CRYPTO_HW_DEQ_FLAG_EXHAUST		(1ULL << 1)
+/**< Bit-mask to indicate dequeuing as many as n jobs in dequeue-many function.
+ *   Without this bit once the driver found out the ready-to-dequeue jobs are
+ *   not as many as n, it shall stop immediate, leave all processed jobs in the
+ *   queue, and return the ready jobs in negative. With this bit set the
+ *   function shall continue dequeue all done jobs and return the dequeued
+ *   job count in positive.
+ */
+
+/**
+ * Typedef for HW direct data-path enqueue callback function.
+ *
+ * @param	qp		Queue pair data.
+ * @param	sess		Cryptodev session.
+ * @param	job		Job data.
+ * @param	opaque		Opaque data to be written to queue descriptor
+ *				when RTE_CRYPTO_HW_ENQ_SET_OPAQUE is
+ *				set.
+ * @param	drv_data	User created temporary driver data for the
+ *				driver to store and update data used between
+ *				adjacent enqueues operations.
+ * @param	flags		Bitmask of RTE_CRYPTO_HW_ENQ_* flags
+ * @return
+ *  - On success return 0
+ *  - On fail return -1
+ **/
+typedef int (*rte_crypto_hw_enq_cb_fn)(void *qp,
+	struct rte_cryptodev_sym_session *sess,
+	struct rte_crypto_sym_job *job, void *opaque, uint64_t *drv_data,
+	uint64_t flags);
+
+/**
+ * Typedef for HW direct data-path dequeue one job callback function.
+ *
+ * @param	qp		Queue pair data.
+ * @param	drv_data	User created temporary driver data for the
+ *				driver to store and update data used between
+ *				adjacent enqueues operations.
+ * @param	flags		Bitmask of RTE_CRYPTO_HW_DEQ_* flags
+ * @param	status		The buffer for the driver to write operation
+ *				status.
+ * @return
+ *  - On success return the opaque data user write in enqueue (if any) and
+ *    - status written as 1 when operation is successful.
+ *    - status written as -1 when operation is failed (e.g. bad MAC)
+ *  - On fail return NULL and status written as 0 when operation is still
+ *    under processing.
+ **/
+typedef void * (*rte_crypto_hw_deq_one_cb_fn)(void *qp, uint64_t *drv_data,
+		uint64_t flags, int *status);
+
+/**
+ * Typedef that the user provided to deal with jobs' status when
+ * dequeue in a bulk.
+ *
+ * @param	data		User provided data.
+ * @param	index		Index number of the processed job.
+ * @param	is_op_success	Driver filled operation status.
+ **/
+typedef void (*rte_crpyto_hw_user_post_deq_cb_fn)(void *data, uint32_t index,
+		uint8_t is_op_success);
+
+/**
+ * Typedef for HW direct data-path dequeue bulk jobs callback function.
+ *
+ * @param	qp		Queue pair data.
+ * @param	drv_data	User created temporary driver data for the
+ *				driver to store and update data used between
+ *				adjacent enqueues operations.
+ * @param	user_data	User provided data to be passed into cb
+ *				function.
+ * @param	cb		User provided callback functions to deal with
+ *				driver returned job status.
+ * @param	n		The number of expected jobs to be dequeued.
+ * @param	flags		Bitmask of RTE_CRYPTO_HW_DEQ_* flags
+ * @param	n_fail		The buffer for driver to write the number of
+ *				failed jobs.
+ * @return
+ *  - Return the number of dequeued jobs.
+ **/
+typedef uint32_t (*rte_crypto_hw_deq_many_cb_fn)(void *qp, uint64_t *drv_data,
+		void *user_data, rte_crpyto_hw_user_post_deq_cb_fn cb,
+		uint32_t n, uint64_t flags, uint32_t *n_fail);
+/**
+ * Typedef for querying HW the number of processed jobs.
+ *
+ * @param	qp		Queue pair data.
+ * @param	nb_jobs		The expected processed job number.
+ * @return
+ *  - If the nb_jobs ready, return 1.
+ *  - Otherwise return 0.
+ **/
+typedef int (*rte_crypto_hw_query_processed)(void *qp, uint32_t nb_jobs);
+
+/* Struct for user to perform HW specific enqueue/dequeue function calls */
+struct rte_crypto_hw_ops {
+	/* Driver written queue pair data pointer, should NOT be alterred by
+	 * the user.
+	 */
+	void *qp;
+	/* Function handler to enqueue AEAD job */
+	rte_crypto_hw_enq_cb_fn enqueue_aead;
+	/* Function handler to enqueue cipher only job */
+	rte_crypto_hw_enq_cb_fn enqueue_cipher;
+	/* Function handler to enqueue auth only job */
+	rte_crypto_hw_enq_cb_fn enqueue_auth;
+	/* Function handler to enqueue cipher + hash chaining job */
+	rte_crypto_hw_enq_cb_fn enqueue_chain;
+	/* Function handler to query processed jobs */
+	rte_crypto_hw_query_processed query_processed;
+	/* Function handler to dequeue one job and return opaque data stored */
+	rte_crypto_hw_deq_one_cb_fn dequeue_one;
+	/* Function handler to dequeue many jobs */
+	rte_crypto_hw_deq_many_cb_fn dequeue_many;
+	/* Reserved */
+	void *reserved[8];
+};
+
+/**
+ * Get the symmetric crypto hardware ops function pointers and queue pair data.
+ *
+ * @param	dev_id	The device identifier.
+ * @param	qp_id	The index of the queue pair from which to retrieve
+ *			processed packets. The value must be in the range
+ *			[0, nb_queue_pair - 1] previously supplied to
+ *			rte_cryptodev_configure().
+ * @param	hw_ops	User provided rte_crypto_hw_ops buffer.
+ *
+ * @return
+ *  - On success hw_ops will be written the HW crypto device's queue pair data
+ *    and function pointers for data enqueue/dequeue.
+ *  - On fail hw_ops is cleared and negative integer is returned.
+ */
+__rte_experimental
+int
+rte_cryptodev_sym_get_hw_ops(
+		uint8_t dev_id, uint16_t qp_id,
+		struct rte_crypto_hw_ops *hw_ops);
+int
+rte_cryptodev_sym_get_hw_ops(
+		uint8_t dev_id, uint16_t qp_id,
+		struct rte_crypto_hw_ops *hw_ops);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.h b/lib/librte_cryptodev/rte_cryptodev_pmd.h
index 81975d72b..28f75d1da 100644
--- a/lib/librte_cryptodev/rte_cryptodev_pmd.h
+++ b/lib/librte_cryptodev/rte_cryptodev_pmd.h
@@ -316,6 +316,10 @@ typedef uint32_t (*cryptodev_sym_cpu_crypto_process_t)
 	(struct rte_cryptodev *dev, struct rte_cryptodev_sym_session *sess,
 	union rte_crypto_sym_ofs ofs, struct rte_crypto_sym_vec *vec);
 
+struct rte_crypto_hw_ops;
+
+typedef int (*cryptodev_sym_hw_get_ops_t)(struct rte_cryptodev *dev,
+		uint16_t qp_id, struct rte_crypto_hw_ops *hw_ops);
 
 /** Crypto device operations function pointer table */
 struct rte_cryptodev_ops {
@@ -348,8 +352,12 @@ struct rte_cryptodev_ops {
 	/**< Clear a Crypto sessions private data. */
 	cryptodev_asym_free_session_t asym_session_clear;
 	/**< Clear a Crypto sessions private data. */
-	cryptodev_sym_cpu_crypto_process_t sym_cpu_process;
-	/**< process input data synchronously (cpu-crypto). */
+	union {
+		cryptodev_sym_cpu_crypto_process_t sym_cpu_process;
+		/**< process input data synchronously (cpu-crypto). */
+		cryptodev_sym_hw_get_ops_t sym_get_hw_ops;
+		/**< Get HW crypto data-path call back functions and data */
+	};
 };
 
 
diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map
index 07a2d2f02..56f5684c8 100644
--- a/lib/librte_cryptodev/rte_cryptodev_version.map
+++ b/lib/librte_cryptodev/rte_cryptodev_version.map
@@ -85,6 +85,7 @@ EXPERIMENTAL {
 	rte_cryptodev_sym_session_set_user_data;
 	rte_crypto_asym_op_strings;
 	rte_crypto_asym_xform_strings;
+	rte_cryptodev_get_qp_status;
 
 	# added in 20.05
 	__rte_cryptodev_trace_configure;
@@ -103,4 +104,7 @@ EXPERIMENTAL {
 	__rte_cryptodev_trace_asym_session_clear;
 	__rte_cryptodev_trace_dequeue_burst;
 	__rte_cryptodev_trace_enqueue_burst;
+
+	# added in 20.08
+	rte_cryptodev_sym_get_hw_ops;
 };
-- 
2.20.1