[dpdk-dev] [RFC PATCH 5/9] crypto/aesni_mb: add rte_security handler

[Fan Zhang <roy.fan.zhang@intel.com>]

This patch add rte_security support support to AESNI-MB PMD. The PMD now
initialize security context instance, create/delete PMD specific security
sessions, and process crypto workloads in synchronous mode.

Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
---
 drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c         | 291 ++++++++++++++++++++-
 drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c     |  91 ++++++-
 drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h |  21 +-
 3 files changed, 398 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
index b495a9679..68767c04e 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
@@ -8,6 +8,8 @@
 #include <rte_hexdump.h>
 #include <rte_cryptodev.h>
 #include <rte_cryptodev_pmd.h>
+#include <rte_security.h>
+#include <rte_security_driver.h>
 #include <rte_bus_vdev.h>
 #include <rte_malloc.h>
 #include <rte_cpuflags.h>
@@ -789,6 +791,167 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 			(UINT64_MAX - u_src + u_dst + 1);
 }
 
+union sec_userdata_field {
+	int status;
+	struct {
+		uint16_t is_gen_digest;
+		uint16_t digest_len;
+	};
+};
+
+struct sec_udata_digest_field {
+	uint32_t is_digest_gen;
+	uint32_t digest_len;
+};
+
+static inline int
+set_mb_job_params_sec(JOB_AES_HMAC *job, struct aesni_mb_sec_session *sec_sess,
+		void *buf, uint32_t buf_len, void *iv, void *aad, void *digest,
+		int *status, uint8_t *digest_idx)
+{
+	struct aesni_mb_session *session = &sec_sess->sess;
+	uint32_t cipher_offset = sec_sess->cipher_offset;
+	void *user_digest = NULL;
+	union sec_userdata_field udata;
+
+	if (unlikely(cipher_offset > buf_len))
+		return -EINVAL;
+
+	/* Set crypto operation */
+	job->chain_order = session->chain_order;
+
+	/* Set cipher parameters */
+	job->cipher_direction = session->cipher.direction;
+	job->cipher_mode = session->cipher.mode;
+
+	job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+
+	/* Set authentication parameters */
+	job->hash_alg = session->auth.algo;
+	job->iv = iv;
+
+	switch (job->hash_alg) {
+	case AES_XCBC:
+		job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
+		job->u.XCBC._k2 = session->auth.xcbc.k2;
+		job->u.XCBC._k3 = session->auth.xcbc.k3;
+
+		job->aes_enc_key_expanded =
+				session->cipher.expanded_aes_keys.encode;
+		job->aes_dec_key_expanded =
+				session->cipher.expanded_aes_keys.decode;
+		break;
+
+	case AES_CCM:
+		job->u.CCM.aad = (uint8_t *)aad + 18;
+		job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
+		job->aes_enc_key_expanded =
+				session->cipher.expanded_aes_keys.encode;
+		job->aes_dec_key_expanded =
+				session->cipher.expanded_aes_keys.decode;
+		job->iv++;
+		break;
+
+	case AES_CMAC:
+		job->u.CMAC._key_expanded = session->auth.cmac.expkey;
+		job->u.CMAC._skey1 = session->auth.cmac.skey1;
+		job->u.CMAC._skey2 = session->auth.cmac.skey2;
+		job->aes_enc_key_expanded =
+				session->cipher.expanded_aes_keys.encode;
+		job->aes_dec_key_expanded =
+				session->cipher.expanded_aes_keys.decode;
+		break;
+
+	case AES_GMAC:
+		if (session->cipher.mode == GCM) {
+			job->u.GCM.aad = aad;
+			job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
+		} else {
+			/* For GMAC */
+			job->u.GCM.aad = aad;
+			job->u.GCM.aad_len_in_bytes = buf_len;
+			job->cipher_mode = GCM;
+		}
+		job->aes_enc_key_expanded = &session->cipher.gcm_key;
+		job->aes_dec_key_expanded = &session->cipher.gcm_key;
+		break;
+
+	default:
+		job->u.HMAC._hashed_auth_key_xor_ipad =
+				session->auth.pads.inner;
+		job->u.HMAC._hashed_auth_key_xor_opad =
+				session->auth.pads.outer;
+
+		if (job->cipher_mode == DES3) {
+			job->aes_enc_key_expanded =
+				session->cipher.exp_3des_keys.ks_ptr;
+			job->aes_dec_key_expanded =
+				session->cipher.exp_3des_keys.ks_ptr;
+		} else {
+			job->aes_enc_key_expanded =
+				session->cipher.expanded_aes_keys.encode;
+			job->aes_dec_key_expanded =
+				session->cipher.expanded_aes_keys.decode;
+		}
+	}
+
+	/* Set digest output location */
+	if (job->hash_alg != NULL_HASH &&
+			session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
+		job->auth_tag_output = sec_sess->temp_digests[*digest_idx];
+		*digest_idx = (*digest_idx + 1) % MAX_JOBS;
+
+		udata.is_gen_digest = 0;
+		udata.digest_len = session->auth.req_digest_len;
+		user_digest = (void *)digest;
+	} else {
+		udata.is_gen_digest = 1;
+		udata.digest_len = session->auth.req_digest_len;
+
+		if (session->auth.req_digest_len !=
+				session->auth.gen_digest_len) {
+			job->auth_tag_output =
+					sec_sess->temp_digests[*digest_idx];
+			*digest_idx = (*digest_idx + 1) % MAX_JOBS;
+
+			user_digest = (void *)digest;
+		} else
+			job->auth_tag_output = digest;
+
+		/* A bit of hack here, since job structure only supports
+		 * 2 user data fields and we need 4 params to be passed
+		 * (status, direction, digest for verify, and length of
+		 * digest), we set the status value as digest length +
+		 * direction here temporarily to avoid creating longer
+		 * buffer to store all 4 params.
+		 */
+		*status = udata.status;
+	}
+	/*
+	 * Multi-buffer library current only support returning a truncated
+	 * digest length as specified in the relevant IPsec RFCs
+	 */
+
+	/* Set digest length */
+	job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
+
+	/* Set IV parameters */
+	job->iv_len_in_bytes = session->iv.length;
+
+	/* Data Parameters */
+	job->src = buf;
+	job->dst = buf;
+	job->cipher_start_src_offset_in_bytes = cipher_offset;
+	job->msg_len_to_cipher_in_bytes = buf_len - cipher_offset;
+	job->hash_start_src_offset_in_bytes = 0;
+	job->msg_len_to_hash_in_bytes = buf_len;
+
+	job->user_data = (void *)status;
+	job->user_data2 = user_digest;
+
+	return 0;
+}
+
 /**
  * Process a crypto operation and complete a JOB_AES_HMAC job structure for
  * submission to the multi buffer library for processing.
@@ -1081,6 +1244,37 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
 	return op;
 }
 
+static inline void
+post_process_mb_sec_job(JOB_AES_HMAC *job)
+{
+	void *user_digest = job->user_data2;
+	int *status = job->user_data;
+	union sec_userdata_field udata;
+
+	switch (job->status) {
+	case STS_COMPLETED:
+		if (user_digest) {
+			udata.status = *status;
+
+			if (udata.is_gen_digest) {
+				*status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+				memcpy(user_digest, job->auth_tag_output,
+						udata.digest_len);
+			} else {
+				verify_digest(job, user_digest,
+					udata.digest_len, (uint8_t *)status);
+
+				if (*status == RTE_CRYPTO_OP_STATUS_AUTH_FAILED)
+					*status = -1;
+			}
+		} else
+			*status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		break;
+	default:
+		*status = RTE_CRYPTO_OP_STATUS_ERROR;
+	}
+}
+
 /**
  * Process a completed JOB_AES_HMAC job and keep processing jobs until
  * get_completed_job return NULL
@@ -1117,6 +1311,32 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
 	return processed_jobs;
 }
 
+static inline uint32_t
+handle_completed_sec_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr)
+{
+	uint32_t processed = 0;
+
+	while (job != NULL) {
+		post_process_mb_sec_job(job);
+		job = IMB_GET_COMPLETED_JOB(mb_mgr);
+		processed++;
+	}
+
+	return processed;
+}
+
+static inline uint32_t
+flush_mb_sec_mgr(MB_MGR *mb_mgr)
+{
+	JOB_AES_HMAC *job = IMB_FLUSH_JOB(mb_mgr);
+	uint32_t processed = 0;
+
+	if (job)
+		processed = handle_completed_sec_jobs(job, mb_mgr);
+
+	return processed;
+}
+
 static inline uint16_t
 flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
 		uint16_t nb_ops)
@@ -1220,6 +1440,55 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	return processed_jobs;
 }
 
+void
+aesni_mb_sec_crypto_process_bulk(struct rte_security_session *sess,
+		struct rte_security_vec buf[], void *iv[], void *aad[],
+		void *digest[], int status[], uint32_t num)
+{
+	struct aesni_mb_sec_session *sec_sess = sess->sess_private_data;
+	JOB_AES_HMAC *job;
+	uint8_t digest_idx = sec_sess->digest_idx;
+	uint32_t i, processed = 0;
+	int ret;
+
+	for (i = 0; i < num; i++) {
+		void *seg_buf = buf[i].vec[0].iov_base;
+		uint32_t buf_len = buf[i].vec[0].iov_len;
+
+		job = IMB_GET_NEXT_JOB(sec_sess->mb_mgr);
+		if (unlikely(job == NULL)) {
+			processed += flush_mb_sec_mgr(sec_sess->mb_mgr);
+
+			job = IMB_GET_NEXT_JOB(sec_sess->mb_mgr);
+			if (!job)
+				return;
+		}
+
+		ret = set_mb_job_params_sec(job, sec_sess, seg_buf, buf_len,
+				iv[i], aad[i], digest[i], &status[i],
+				&digest_idx);
+				/* Submit job to multi-buffer for processing */
+		if (ret) {
+			processed++;
+			status[i] = ret;
+			continue;
+		}
+
+#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
+		job = IMB_SUBMIT_JOB(sec_sess->mb_mgr);
+#else
+		job = IMB_SUBMIT_JOB_NOCHECK(sec_sess->mb_mgr);
+#endif
+
+		if (job)
+			processed += handle_completed_sec_jobs(job,
+					sec_sess->mb_mgr);
+	}
+
+	while (processed < num)
+		processed += flush_mb_sec_mgr(sec_sess->mb_mgr);
+}
+
 static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
 
 static int
@@ -1229,8 +1498,10 @@ cryptodev_aesni_mb_create(const char *name,
 {
 	struct rte_cryptodev *dev;
 	struct aesni_mb_private *internals;
+	struct rte_security_ctx *sec_ctx;
 	enum aesni_mb_vector_mode vector_mode;
 	MB_MGR *mb_mgr;
+	char sec_name[RTE_DEV_NAME_MAX_LEN];
 
 	/* Check CPU for support for AES instruction set */
 	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
@@ -1264,7 +1535,8 @@ cryptodev_aesni_mb_create(const char *name,
 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			RTE_CRYPTODEV_FF_CPU_AESNI |
-			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
+			RTE_CRYPTODEV_FF_SECURITY;
 
 
 	mb_mgr = alloc_mb_mgr(0);
@@ -1303,11 +1575,28 @@ cryptodev_aesni_mb_create(const char *name,
 	AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
 			imb_get_version_str());
 
+	/* setup security operations */
+	snprintf(sec_name, sizeof(sec_name) - 1, "aes_mb_sec_%u",
+			dev->driver_id);
+	sec_ctx = rte_zmalloc_socket(sec_name,
+			sizeof(struct rte_security_ctx),
+			RTE_CACHE_LINE_SIZE, init_params->socket_id);
+	if (sec_ctx == NULL) {
+		AESNI_MB_LOG(ERR, "memory allocation failed\n");
+		goto error_exit;
+	}
+
+	sec_ctx->device = (void *)dev;
+	sec_ctx->ops = rte_aesni_mb_pmd_security_ops;
+	dev->security_ctx = sec_ctx;
+
 	return 0;
 
 error_exit:
 	if (mb_mgr)
 		free_mb_mgr(mb_mgr);
+	if (sec_ctx)
+		rte_free(sec_ctx);
 
 	rte_cryptodev_pmd_destroy(dev);
 
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
index 8d15b99d4..ca6cea775 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
@@ -8,6 +8,7 @@
 #include <rte_common.h>
 #include <rte_malloc.h>
 #include <rte_cryptodev_pmd.h>
+#include <rte_security_driver.h>
 
 #include "rte_aesni_mb_pmd_private.h"
 
@@ -732,7 +733,8 @@ aesni_mb_pmd_qp_count(struct rte_cryptodev *dev)
 static unsigned
 aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 {
-	return sizeof(struct aesni_mb_session);
+	return RTE_ALIGN_CEIL(sizeof(struct aesni_mb_session),
+			RTE_CACHE_LINE_SIZE);
 }
 
 /** Configure a aesni multi-buffer session from a crypto xform chain */
@@ -810,4 +812,91 @@ struct rte_cryptodev_ops aesni_mb_pmd_ops = {
 		.sym_session_clear	= aesni_mb_pmd_sym_session_clear
 };
 
+/** Set session authentication parameters */
+
+static int
+aesni_mb_security_session_create(void *dev,
+		struct rte_security_session_conf *conf,
+		struct rte_security_session *sess,
+		struct rte_mempool *mempool)
+{
+	struct rte_cryptodev *cdev = dev;
+	struct aesni_mb_private *internals = cdev->data->dev_private;
+	struct aesni_mb_sec_session *sess_priv;
+	int ret;
+
+	if (!conf->crypto_xform) {
+		AESNI_MB_LOG(ERR, "Invalid security session conf");
+		return -EINVAL;
+	}
+
+	if (rte_mempool_get(mempool, (void **)(&sess_priv))) {
+		AESNI_MB_LOG(ERR,
+				"Couldn't get object from session mempool");
+		return -ENOMEM;
+	}
+
+	sess_priv->mb_mgr = internals->mb_mgr;
+	if (sess_priv->mb_mgr == NULL)
+		return -ENOMEM;
+
+	sess_priv->cipher_offset = conf->cpucrypto.cipher_offset;
+
+	ret = aesni_mb_set_session_parameters(sess_priv->mb_mgr,
+			&sess_priv->sess, conf->crypto_xform);
+	if (ret != 0) {
+		AESNI_MB_LOG(ERR, "failed configure session parameters");
+
+		rte_mempool_put(mempool, sess_priv);
+	}
+
+	sess->sess_private_data = (void *)sess_priv;
+
+	return ret;
+}
+
+static int
+aesni_mb_security_session_destroy(void *dev __rte_unused,
+		struct rte_security_session *sess)
+{
+	struct aesni_mb_sec_session *sess_priv =
+			get_sec_session_private_data(sess);
+
+	if (sess_priv) {
+		struct rte_mempool *sess_mp = rte_mempool_from_obj(
+				(void *)sess_priv);
+
+		memset(sess, 0, sizeof(struct aesni_mb_sec_session));
+		set_sec_session_private_data(sess, NULL);
+
+		if (sess_mp == NULL) {
+			AESNI_MB_LOG(ERR, "failed fetch session mempool");
+			return -EINVAL;
+		}
+
+		rte_mempool_put(sess_mp, sess_priv);
+	}
+
+	return 0;
+}
+
+static unsigned int
+aesni_mb_sec_session_get_size(__rte_unused void *device)
+{
+	return RTE_ALIGN_CEIL(sizeof(struct aesni_mb_sec_session),
+			RTE_CACHE_LINE_SIZE);
+}
+
+static struct rte_security_ops aesni_mb_security_ops = {
+		.session_create = aesni_mb_security_session_create,
+		.session_get_size = aesni_mb_sec_session_get_size,
+		.session_update = NULL,
+		.session_stats_get = NULL,
+		.session_destroy = aesni_mb_security_session_destroy,
+		.set_pkt_metadata = NULL,
+		.capabilities_get = NULL,
+		.process_cpu_crypto_bulk = aesni_mb_sec_crypto_process_bulk,
+};
+
 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
+struct rte_security_ops *rte_aesni_mb_pmd_security_ops = &aesni_mb_security_ops;
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
index b794d4bc1..d1cf416ab 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
@@ -176,7 +176,6 @@ struct aesni_mb_qp {
 	 */
 } __rte_cache_aligned;
 
-/** AES-NI multi-buffer private session structure */
 struct aesni_mb_session {
 	JOB_CHAIN_ORDER chain_order;
 	struct {
@@ -265,16 +264,32 @@ struct aesni_mb_session {
 		/** AAD data length */
 		uint16_t aad_len;
 	} aead;
-} __rte_cache_aligned;
+};
+
+/** AES-NI multi-buffer private security session structure */
+struct aesni_mb_sec_session {
+	/**< Unique Queue Pair Name */
+	struct aesni_mb_session sess;
+	uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];
+	uint16_t digest_idx;
+	uint32_t cipher_offset;
+	MB_MGR *mb_mgr;
+};
 
 extern int
 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 		struct aesni_mb_session *sess,
 		const struct rte_crypto_sym_xform *xform);
 
+extern void
+aesni_mb_sec_crypto_process_bulk(struct rte_security_session *sess,
+		struct rte_security_vec buf[], void *iv[], void *aad[],
+		void *digest[], int status[], uint32_t num);
+
 /** device specific operations function pointer structure */
 extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;
 
-
+/** device specific operations function pointer structure for rte_security */
+extern struct rte_security_ops *rte_aesni_mb_pmd_security_ops;
 
 #endif /* _RTE_AESNI_MB_PMD_PRIVATE_H_ */
-- 
2.14.5