[PATCH 4/5] crypto/uadk: support auth algorithms

[Zhangfei Gao <zhangfei.gao@linaro.org>]

Hash algorithms:

* ``RTE_CRYPTO_AUTH_MD5``
* ``RTE_CRYPTO_AUTH_MD5_HMAC``
* ``RTE_CRYPTO_AUTH_SHA1``
* ``RTE_CRYPTO_AUTH_SHA1_HMAC``
* ``RTE_CRYPTO_AUTH_SHA224``
* ``RTE_CRYPTO_AUTH_SHA224_HMAC``
* ``RTE_CRYPTO_AUTH_SHA256``
* ``RTE_CRYPTO_AUTH_SHA256_HMAC``
* ``RTE_CRYPTO_AUTH_SHA384``
* ``RTE_CRYPTO_AUTH_SHA384_HMAC``
* ``RTE_CRYPTO_AUTH_SHA512``
* ``RTE_CRYPTO_AUTH_SHA512_HMAC``

Signed-off-by: Zhangfei Gao <zhangfei.gao@linaro.org>
---
 drivers/crypto/uadk/uadk_crypto_pmd.c | 461 ++++++++++++++++++++++++++
 1 file changed, 461 insertions(+)

diff --git a/drivers/crypto/uadk/uadk_crypto_pmd.c b/drivers/crypto/uadk/uadk_crypto_pmd.c
index 7ca3e3ecd8..a1196560b8 100644
--- a/drivers/crypto/uadk/uadk_crypto_pmd.c
+++ b/drivers/crypto/uadk/uadk_crypto_pmd.c
@@ -31,11 +31,15 @@ struct uadk_qp {
 
 enum uadk_chain_order {
 	UADK_CHAIN_ONLY_CIPHER,
+	UADK_CHAIN_ONLY_AUTH,
+	UADK_CHAIN_CIPHER_AUTH,
+	UADK_CHAIN_AUTH_CIPHER,
 	UADK_CHAIN_NOT_SUPPORTED
 };
 
 struct uadk_crypto_session {
 	handle_t handle_cipher;
+	handle_t handle_digest;
 	enum uadk_chain_order chain_order;
 
 	/* IV parameters */
@@ -50,6 +54,15 @@ struct uadk_crypto_session {
 		/* cipher operation direction */
 		struct wd_cipher_req req;
 	} cipher;
+
+	/* Authentication Parameters */
+	struct {
+		struct wd_digest_req req;
+		enum rte_crypto_auth_operation operation;
+		/* auth operation generate or verify */
+		uint16_t digest_length;
+		/* digest length */
+	} auth;
 } __rte_cache_aligned;
 
 enum uadk_supported_platform {
@@ -59,6 +72,7 @@ enum uadk_supported_platform {
 
 struct uadk_crypto_priv {
 	bool env_cipher_init;
+	bool env_auth_init;
 	enum uadk_supported_platform platform;
 } __rte_cache_aligned;
 
@@ -72,6 +86,252 @@ RTE_LOG_REGISTER_DEFAULT(uadk_crypto_logtype, INFO);
 		## __VA_ARGS__)
 
 static const struct rte_cryptodev_capabilities uadk_crypto_920_capabilities[] = {
+	{	/* MD5 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_MD5_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* MD5 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_MD5,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA1 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 20,
+					.max = 20,
+					.increment = 0
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA1 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA1,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 20,
+					.max = 20,
+					.increment = 0
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA224 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 28,
+					.max = 28,
+					.increment = 0
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA224 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA224,
+				.block_size = 64,
+					.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 28,
+					.max = 28,
+					.increment = 0
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA256 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 32,
+					.max = 32,
+					.increment = 0
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA256 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA256,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 32,
+					.max = 32,
+					.increment = 0
+				},
+			}, }
+		}, }
+	},
+	{	/* SHA384 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
+				.block_size = 128,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 48,
+					.max = 48,
+					.increment = 0
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA384 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA384,
+				.block_size = 64,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 48,
+					.max = 48,
+					.increment = 0
+					},
+			}, }
+		}, }
+	},
+	{	/* SHA512 HMAC */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
+				.block_size = 128,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 64,
+					.max = 64,
+					.increment = 0
+				},
+				.iv_size = { 0 }
+			}, }
+		}, }
+	},
+	{	/* SHA512 */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+			{.auth = {
+				.algo = RTE_CRYPTO_AUTH_SHA512,
+				.block_size = 128,
+				.key_size = {
+					.min = 0,
+					.max = 0,
+					.increment = 0
+				},
+				.digest_size = {
+					.min = 64,
+					.max = 64,
+					.increment = 0
+				},
+			}, }
+		}, }
+	},
 	{	/* AES ECB */
 		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 		{.sym = {
@@ -188,6 +448,11 @@ uadk_crypto_pmd_close(struct rte_cryptodev *dev)
 		priv->env_cipher_init = false;
 	}
 
+	if (priv->env_auth_init) {
+		wd_digest_env_uninit();
+		priv->env_auth_init = false;
+	}
+
 	return 0;
 }
 
@@ -346,9 +611,19 @@ uadk_get_chain_order(const struct rte_crypto_sym_xform *xform)
 	enum uadk_chain_order res = UADK_CHAIN_NOT_SUPPORTED;
 
 	if (xform != NULL) {
+		if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+			if (xform->next == NULL)
+				res = UADK_CHAIN_ONLY_AUTH;
+			else if (xform->next->type ==
+					RTE_CRYPTO_SYM_XFORM_CIPHER)
+				res = UADK_CHAIN_AUTH_CIPHER;
+		}
+
 		if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 			if (xform->next == NULL)
 				res = UADK_CHAIN_ONLY_CIPHER;
+			else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
+				res = UADK_CHAIN_CIPHER_AUTH;
 		}
 	}
 
@@ -435,6 +710,112 @@ uadk_set_session_cipher_parameters(struct rte_cryptodev *dev,
 	return ret;
 }
 
+/* Set session auth parameters */
+static int
+uadk_set_session_auth_parameters(struct rte_cryptodev *dev,
+				 struct uadk_crypto_session *sess,
+				 struct rte_crypto_sym_xform *xform)
+{
+	struct uadk_crypto_priv *priv = dev->data->dev_private;
+	struct wd_digest_sess_setup setup = {0};
+	struct sched_params params = {0};
+	int ret;
+
+	if (!priv->env_auth_init) {
+		ret = wd_digest_env_init(NULL);
+		if (ret < 0)
+			return -EINVAL;
+		priv->env_auth_init = true;
+	}
+
+	sess->auth.operation = xform->auth.op;
+	sess->auth.digest_length = xform->auth.digest_length;
+
+	switch (xform->auth.algo) {
+	case RTE_CRYPTO_AUTH_MD5:
+	case RTE_CRYPTO_AUTH_MD5_HMAC:
+		setup.mode = (xform->auth.algo == RTE_CRYPTO_AUTH_MD5) ?
+			     WD_DIGEST_NORMAL : WD_DIGEST_HMAC;
+		setup.alg = WD_DIGEST_MD5;
+		sess->auth.req.out_buf_bytes = 16;
+		sess->auth.req.out_bytes = 16;
+		break;
+	case RTE_CRYPTO_AUTH_SHA1:
+	case RTE_CRYPTO_AUTH_SHA1_HMAC:
+		setup.mode = (xform->auth.algo == RTE_CRYPTO_AUTH_SHA1) ?
+			     WD_DIGEST_NORMAL : WD_DIGEST_HMAC;
+		setup.alg = WD_DIGEST_SHA1;
+		sess->auth.req.out_buf_bytes = 20;
+		sess->auth.req.out_bytes = 20;
+		break;
+	case RTE_CRYPTO_AUTH_SHA224:
+	case RTE_CRYPTO_AUTH_SHA224_HMAC:
+		setup.mode = (xform->auth.algo == RTE_CRYPTO_AUTH_SHA224) ?
+			     WD_DIGEST_NORMAL : WD_DIGEST_HMAC;
+		setup.alg = WD_DIGEST_SHA224;
+		sess->auth.req.out_buf_bytes = 28;
+		sess->auth.req.out_bytes = 28;
+		break;
+	case RTE_CRYPTO_AUTH_SHA256:
+	case RTE_CRYPTO_AUTH_SHA256_HMAC:
+		setup.mode = (xform->auth.algo == RTE_CRYPTO_AUTH_SHA256) ?
+			     WD_DIGEST_NORMAL : WD_DIGEST_HMAC;
+		setup.alg = WD_DIGEST_SHA256;
+		sess->auth.req.out_buf_bytes = 32;
+		sess->auth.req.out_bytes = 32;
+		break;
+	case RTE_CRYPTO_AUTH_SHA384:
+	case RTE_CRYPTO_AUTH_SHA384_HMAC:
+		setup.mode = (xform->auth.algo == RTE_CRYPTO_AUTH_SHA384) ?
+			     WD_DIGEST_NORMAL : WD_DIGEST_HMAC;
+		setup.alg = WD_DIGEST_SHA384;
+		sess->auth.req.out_buf_bytes = 48;
+		sess->auth.req.out_bytes = 48;
+		break;
+	case RTE_CRYPTO_AUTH_SHA512:
+	case RTE_CRYPTO_AUTH_SHA512_HMAC:
+		setup.mode = (xform->auth.algo == RTE_CRYPTO_AUTH_SHA512) ?
+			     WD_DIGEST_NORMAL : WD_DIGEST_HMAC;
+		setup.alg = WD_DIGEST_SHA512;
+		sess->auth.req.out_buf_bytes = 64;
+		sess->auth.req.out_bytes = 64;
+		break;
+	default:
+		ret = -ENOTSUP;
+		goto env_uninit;
+	}
+
+	params.numa_id = -1;	/* choose nearby numa node */
+	setup.sched_param = &params;
+	sess->handle_digest = wd_digest_alloc_sess(&setup);
+	if (!sess->handle_digest) {
+		UADK_LOG(ERR, "uadk failed to alloc session!\n");
+		ret = -EINVAL;
+		goto env_uninit;
+	}
+
+	/* if mode is HMAC, should set key */
+	if (setup.mode == WD_DIGEST_HMAC) {
+		ret = wd_digest_set_key(sess->handle_digest,
+					xform->auth.key.data,
+					xform->auth.key.length);
+		if (ret) {
+			UADK_LOG(ERR, "uadk failed to alloc session!\n");
+			wd_digest_free_sess(sess->handle_digest);
+			sess->handle_digest = 0;
+			ret = -EINVAL;
+			goto env_uninit;
+		}
+	}
+
+	return 0;
+
+env_uninit:
+	wd_digest_env_uninit();
+	priv->env_auth_init = false;
+	return ret;
+}
+
 static int
 uadk_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
@@ -442,6 +823,7 @@ uadk_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_mempool *mp)
 {
 	struct rte_crypto_sym_xform *cipher_xform = NULL;
+	struct rte_crypto_sym_xform *auth_xform = NULL;
 	struct uadk_crypto_session *sess;
 	int ret;
 
@@ -457,6 +839,17 @@ uadk_crypto_sym_session_configure(struct rte_cryptodev *dev,
 	case UADK_CHAIN_ONLY_CIPHER:
 		cipher_xform = xform;
 		break;
+	case UADK_CHAIN_ONLY_AUTH:
+		auth_xform = xform;
+		break;
+	case UADK_CHAIN_CIPHER_AUTH:
+		cipher_xform = xform;
+		auth_xform = xform->next;
+		break;
+	case UADK_CHAIN_AUTH_CIPHER:
+		auth_xform = xform;
+		cipher_xform = xform->next;
+		break;
 	default:
 		ret = -ENOTSUP;
 		goto err;
@@ -471,6 +864,15 @@ uadk_crypto_sym_session_configure(struct rte_cryptodev *dev,
 		}
 	}
 
+	if (auth_xform) {
+		ret = uadk_set_session_auth_parameters(dev, sess, auth_xform);
+		if (ret != 0) {
+			UADK_LOG(ERR,
+				"Invalid/unsupported auth parameters");
+			goto err;
+		}
+	}
+
 	set_sym_session_private_data(session, dev->driver_id, sess);
 
 	return 0;
@@ -496,6 +898,11 @@ uadk_crypto_sym_session_clear(struct rte_cryptodev *dev,
 		priv_sess->handle_cipher = 0;
 	}
 
+	if (priv_sess->handle_digest) {
+		wd_digest_free_sess(priv_sess->handle_digest);
+		priv_sess->handle_digest = 0;
+	}
+
 	set_sym_session_private_data(sess, dev->driver_id, NULL);
 	rte_mempool_put(rte_mempool_from_obj(priv_sess), priv_sess);
 }
@@ -551,6 +958,49 @@ uadk_process_cipher_op(struct rte_crypto_op *op,
 		op->status = RTE_COMP_OP_STATUS_ERROR;
 }
 
+static void
+uadk_process_auth_op(struct uadk_qp *qp, struct rte_crypto_op *op,
+		     struct uadk_crypto_session *sess,
+		     struct rte_mbuf *msrc, struct rte_mbuf *mdst)
+{
+	uint32_t srclen = op->sym->auth.data.length;
+	uint32_t off = op->sym->auth.data.offset;
+	uint8_t *dst = qp->temp_digest;
+	int ret;
+
+	if (!sess) {
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return;
+	}
+
+	sess->auth.req.in = rte_pktmbuf_mtod_offset(msrc, uint8_t *, off);
+	sess->auth.req.in_bytes = srclen;
+	sess->auth.req.out = dst;
+
+	do {
+		ret = wd_do_digest_sync(sess->handle_digest, &sess->auth.req);
+	} while (ret == -WD_EBUSY);
+
+	if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
+		if (memcmp(dst, op->sym->auth.digest.data,
+				sess->auth.digest_length) != 0) {
+			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+		}
+	} else {
+		uint8_t *auth_dst;
+
+		auth_dst = op->sym->auth.digest.data;
+		if (auth_dst == NULL)
+			auth_dst = rte_pktmbuf_mtod_offset(mdst, uint8_t *,
+					op->sym->auth.data.offset +
+					op->sym->auth.data.length);
+		memcpy(auth_dst, dst, sess->auth.digest_length);
+	}
+
+	if (ret)
+		op->status = RTE_COMP_OP_STATUS_ERROR;
+}
+
 static uint16_t
 uadk_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 			  uint16_t nb_ops)
@@ -580,6 +1030,17 @@ uadk_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		case UADK_CHAIN_ONLY_CIPHER:
 			uadk_process_cipher_op(op, sess, msrc, mdst);
 			break;
+		case UADK_CHAIN_ONLY_AUTH:
+			uadk_process_auth_op(qp, op, sess, msrc, mdst);
+			break;
+		case UADK_CHAIN_CIPHER_AUTH:
+			uadk_process_cipher_op(op, sess, msrc, mdst);
+			uadk_process_auth_op(qp, op, sess, mdst, mdst);
+			break;
+		case UADK_CHAIN_AUTH_CIPHER:
+			uadk_process_auth_op(qp, op, sess, msrc, mdst);
+			uadk_process_cipher_op(op, sess, msrc, mdst);
+			break;
 		default:
 			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 			break;
-- 
2.36.1