[dpdk-dev] [PATCH v2 1/4] crypto/cnxk: add asymmetric session ops

[Anoob Joseph <anoobj@marvell.com>]

From: Kiran Kumar K <kirankumark@marvell.com>

Add asymmetric crypto session ops.

Signed-off-by: Kiran Kumar K <kirankumark@marvell.com>
---
 doc/guides/cryptodevs/features/cn10k.ini  |  13 ++
 doc/guides/cryptodevs/features/cn9k.ini   |  13 ++
 drivers/crypto/cnxk/cn10k_cryptodev.c     |   2 +
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c |   6 +-
 drivers/crypto/cnxk/cn9k_cryptodev.c      |   4 +-
 drivers/crypto/cnxk/cn9k_cryptodev_ops.c  |   6 +-
 drivers/crypto/cnxk/cnxk_ae.h             | 211 ++++++++++++++++++++++++++++++
 drivers/crypto/cnxk/cnxk_cryptodev.h      |   4 +-
 drivers/crypto/cnxk/cnxk_cryptodev_ops.c  | 106 +++++++++++++++
 drivers/crypto/cnxk/cnxk_cryptodev_ops.h  |   8 ++
 10 files changed, 365 insertions(+), 8 deletions(-)
 create mode 100644 drivers/crypto/cnxk/cnxk_ae.h

diff --git a/doc/guides/cryptodevs/features/cn10k.ini b/doc/guides/cryptodevs/features/cn10k.ini
index b268f84..f5552fe 100644
--- a/doc/guides/cryptodevs/features/cn10k.ini
+++ b/doc/guides/cryptodevs/features/cn10k.ini
@@ -5,6 +5,7 @@
 ;
 [Features]
 Symmetric crypto       = Y
+Asymmetric crypto      = Y
 Sym operation chaining = Y
 HW Accelerated         = Y
 Protocol offload       = Y
@@ -65,3 +66,15 @@ AES GCM (128)     = Y
 AES GCM (192)     = Y
 AES GCM (256)     = Y
 CHACHA20-POLY1305 = Y
+
+;
+; Supported Asymmetric algorithms of the 'cn10k' crypto driver.
+;
+[Asymmetric]
+RSA                     = Y
+DSA                     =
+Modular Exponentiation  = Y
+Modular Inversion       =
+Diffie-hellman          =
+ECDSA                   = Y
+ECPM                    = Y
diff --git a/doc/guides/cryptodevs/features/cn9k.ini b/doc/guides/cryptodevs/features/cn9k.ini
index 7b310e6..d69dbe8 100644
--- a/doc/guides/cryptodevs/features/cn9k.ini
+++ b/doc/guides/cryptodevs/features/cn9k.ini
@@ -5,6 +5,7 @@
 ;
 [Features]
 Symmetric crypto       = Y
+Asymmetric crypto      = Y
 Sym operation chaining = Y
 HW Accelerated         = Y
 In Place SGL           = Y
@@ -64,3 +65,15 @@ AES GCM (128)     = Y
 AES GCM (192)     = Y
 AES GCM (256)     = Y
 CHACHA20-POLY1305 = Y
+
+;
+; Supported Asymmetric algorithms of the 'cn9k' crypto driver.
+;
+[Asymmetric]
+RSA                     = Y
+DSA                     =
+Modular Exponentiation  = Y
+Modular Inversion       =
+Diffie-hellman          =
+ECDSA                   = Y
+ECPM                    = Y
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev.c b/drivers/crypto/cnxk/cn10k_cryptodev.c
index 22ae810..10a621f 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev.c
@@ -92,7 +92,9 @@ cn10k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	dev->driver_id = cn10k_cryptodev_driver_id;
 
 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			     RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
 			     RTE_CRYPTODEV_FF_HW_ACCELERATED |
+			     RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT |
 			     RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			     RTE_CRYPTODEV_FF_IN_PLACE_SGL |
 			     RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 29525cd..8b33764 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -426,8 +426,8 @@ struct rte_cryptodev_ops cn10k_cpt_ops = {
 	.sym_session_clear = cnxk_cpt_sym_session_clear,
 
 	/* Asymmetric crypto ops */
-	.asym_session_get_size = NULL,
-	.asym_session_configure = NULL,
-	.asym_session_clear = NULL,
+	.asym_session_get_size = cnxk_ae_session_size_get,
+	.asym_session_configure = cnxk_ae_session_cfg,
+	.asym_session_clear = cnxk_ae_session_clear,
 
 };
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev.c b/drivers/crypto/cnxk/cn9k_cryptodev.c
index d3dc084..e74e739 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev.c
@@ -83,6 +83,7 @@ cn9k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	cnxk_cpt_caps_populate(vf);
 
 	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			     RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
 			     RTE_CRYPTODEV_FF_HW_ACCELERATED |
 			     RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			     RTE_CRYPTODEV_FF_IN_PLACE_SGL |
@@ -90,7 +91,8 @@ cn9k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 			     RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
 			     RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
 			     RTE_CRYPTODEV_FF_SYM_SESSIONLESS |
-			     RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED;
+			     RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED |
+			     RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT;
 
 	cn9k_cpt_set_enqdeq_fns(dev);
 
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index da13c7d..d8b2aea 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -312,8 +312,8 @@ struct rte_cryptodev_ops cn9k_cpt_ops = {
 	.sym_session_clear = cnxk_cpt_sym_session_clear,
 
 	/* Asymmetric crypto ops */
-	.asym_session_get_size = NULL,
-	.asym_session_configure = NULL,
-	.asym_session_clear = NULL,
+	.asym_session_get_size = cnxk_ae_session_size_get,
+	.asym_session_configure = cnxk_ae_session_cfg,
+	.asym_session_clear = cnxk_ae_session_clear,
 
 };
diff --git a/drivers/crypto/cnxk/cnxk_ae.h b/drivers/crypto/cnxk/cnxk_ae.h
new file mode 100644
index 0000000..e3dd63b
--- /dev/null
+++ b/drivers/crypto/cnxk/cnxk_ae.h
@@ -0,0 +1,211 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#ifndef _CNXK_AE_H_
+#define _CNXK_AE_H_
+
+#include <rte_common.h>
+#include <rte_crypto_asym.h>
+#include <rte_malloc.h>
+
+#include "roc_api.h"
+#include "cnxk_cryptodev_ops.h"
+
+struct cnxk_ae_sess {
+	enum rte_crypto_asym_xform_type xfrm_type;
+	union {
+		struct rte_crypto_rsa_xform rsa_ctx;
+		struct rte_crypto_modex_xform mod_ctx;
+		struct roc_ae_ec_ctx ec_ctx;
+	};
+	uint64_t *cnxk_fpm_iova;
+	struct roc_ae_ec_group **ec_grp;
+	uint64_t cpt_inst_w7;
+};
+
+static __rte_always_inline void
+cnxk_ae_modex_param_normalize(uint8_t **data, size_t *len)
+{
+	size_t i;
+
+	/* Strip leading NUL bytes */
+	for (i = 0; i < *len; i++) {
+		if ((*data)[i] != 0)
+			break;
+	}
+	*data += i;
+	*len -= i;
+}
+
+static __rte_always_inline int
+cnxk_ae_fill_modex_params(struct cnxk_ae_sess *sess,
+			  struct rte_crypto_asym_xform *xform)
+{
+	struct rte_crypto_modex_xform *ctx = &sess->mod_ctx;
+	size_t exp_len = xform->modex.exponent.length;
+	size_t mod_len = xform->modex.modulus.length;
+	uint8_t *exp = xform->modex.exponent.data;
+	uint8_t *mod = xform->modex.modulus.data;
+
+	cnxk_ae_modex_param_normalize(&mod, &mod_len);
+	cnxk_ae_modex_param_normalize(&exp, &exp_len);
+
+	if (unlikely(exp_len == 0 || mod_len == 0))
+		return -EINVAL;
+
+	if (unlikely(exp_len > mod_len))
+		return -ENOTSUP;
+
+	/* Allocate buffer to hold modexp params */
+	ctx->modulus.data = rte_malloc(NULL, mod_len + exp_len, 0);
+	if (ctx->modulus.data == NULL)
+		return -ENOMEM;
+
+	/* Set up modexp prime modulus and private exponent */
+	memcpy(ctx->modulus.data, mod, mod_len);
+	ctx->exponent.data = ctx->modulus.data + mod_len;
+	memcpy(ctx->exponent.data, exp, exp_len);
+
+	ctx->modulus.length = mod_len;
+	ctx->exponent.length = exp_len;
+
+	return 0;
+}
+
+static __rte_always_inline int
+cnxk_ae_fill_rsa_params(struct cnxk_ae_sess *sess,
+			struct rte_crypto_asym_xform *xform)
+{
+	struct rte_crypto_rsa_priv_key_qt qt = xform->rsa.qt;
+	struct rte_crypto_rsa_xform *xfrm_rsa = &xform->rsa;
+	struct rte_crypto_rsa_xform *rsa = &sess->rsa_ctx;
+	size_t mod_len = xfrm_rsa->n.length;
+	size_t exp_len = xfrm_rsa->e.length;
+	size_t len = (mod_len / 2);
+	uint64_t total_size;
+
+	/* Make sure key length used is not more than mod_len/2 */
+	if (qt.p.data != NULL)
+		len = RTE_MIN(len, qt.p.length);
+
+	/* Total size required for RSA key params(n,e,(q,dQ,p,dP,qInv)) */
+	total_size = mod_len + exp_len + 5 * len;
+
+	/* Allocate buffer to hold all RSA keys */
+	rsa->n.data = rte_malloc(NULL, total_size, 0);
+	if (rsa->n.data == NULL)
+		return -ENOMEM;
+
+	/* Set up RSA prime modulus and public key exponent */
+	memcpy(rsa->n.data, xfrm_rsa->n.data, mod_len);
+	rsa->e.data = rsa->n.data + mod_len;
+	memcpy(rsa->e.data, xfrm_rsa->e.data, exp_len);
+
+	/* Private key in quintuple format */
+	if (len != 0) {
+		rsa->qt.q.data = rsa->e.data + exp_len;
+		memcpy(rsa->qt.q.data, qt.q.data, qt.q.length);
+		rsa->qt.dQ.data = rsa->qt.q.data + qt.q.length;
+		memcpy(rsa->qt.dQ.data, qt.dQ.data, qt.dQ.length);
+		rsa->qt.p.data = rsa->qt.dQ.data + qt.dQ.length;
+		memcpy(rsa->qt.p.data, qt.p.data, qt.p.length);
+		rsa->qt.dP.data = rsa->qt.p.data + qt.p.length;
+		memcpy(rsa->qt.dP.data, qt.dP.data, qt.dP.length);
+		rsa->qt.qInv.data = rsa->qt.dP.data + qt.dP.length;
+		memcpy(rsa->qt.qInv.data, qt.qInv.data, qt.qInv.length);
+
+		rsa->qt.q.length = qt.q.length;
+		rsa->qt.dQ.length = qt.dQ.length;
+		rsa->qt.p.length = qt.p.length;
+		rsa->qt.dP.length = qt.dP.length;
+		rsa->qt.qInv.length = qt.qInv.length;
+	}
+	rsa->n.length = mod_len;
+	rsa->e.length = exp_len;
+
+	return 0;
+}
+
+static __rte_always_inline int
+cnxk_ae_fill_ec_params(struct cnxk_ae_sess *sess,
+		       struct rte_crypto_asym_xform *xform)
+{
+	struct roc_ae_ec_ctx *ec = &sess->ec_ctx;
+
+	switch (xform->ec.curve_id) {
+	case RTE_CRYPTO_EC_GROUP_SECP192R1:
+		ec->curveid = ROC_AE_EC_ID_P192;
+		break;
+	case RTE_CRYPTO_EC_GROUP_SECP224R1:
+		ec->curveid = ROC_AE_EC_ID_P224;
+		break;
+	case RTE_CRYPTO_EC_GROUP_SECP256R1:
+		ec->curveid = ROC_AE_EC_ID_P256;
+		break;
+	case RTE_CRYPTO_EC_GROUP_SECP384R1:
+		ec->curveid = ROC_AE_EC_ID_P384;
+		break;
+	case RTE_CRYPTO_EC_GROUP_SECP521R1:
+		ec->curveid = ROC_AE_EC_ID_P521;
+		break;
+	default:
+		/* Only NIST curves (FIPS 186-4) are supported */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static __rte_always_inline int
+cnxk_ae_fill_session_parameters(struct cnxk_ae_sess *sess,
+				struct rte_crypto_asym_xform *xform)
+{
+	int ret;
+
+	sess->xfrm_type = xform->xform_type;
+
+	switch (xform->xform_type) {
+	case RTE_CRYPTO_ASYM_XFORM_RSA:
+		ret = cnxk_ae_fill_rsa_params(sess, xform);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_MODEX:
+		ret = cnxk_ae_fill_modex_params(sess, xform);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_ECDSA:
+		/* Fall through */
+	case RTE_CRYPTO_ASYM_XFORM_ECPM:
+		ret = cnxk_ae_fill_ec_params(sess, xform);
+		break;
+	default:
+		return -ENOTSUP;
+	}
+	return ret;
+}
+
+static inline void
+cnxk_ae_free_session_parameters(struct cnxk_ae_sess *sess)
+{
+	struct rte_crypto_modex_xform *mod;
+	struct rte_crypto_rsa_xform *rsa;
+
+	switch (sess->xfrm_type) {
+	case RTE_CRYPTO_ASYM_XFORM_RSA:
+		rsa = &sess->rsa_ctx;
+		if (rsa->n.data)
+			rte_free(rsa->n.data);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_MODEX:
+		mod = &sess->mod_ctx;
+		if (mod->modulus.data)
+			rte_free(mod->modulus.data);
+		break;
+	case RTE_CRYPTO_ASYM_XFORM_ECDSA:
+		/* Fall through */
+	case RTE_CRYPTO_ASYM_XFORM_ECPM:
+		break;
+	default:
+		break;
+	}
+}
+#endif /* _CNXK_AE_H_ */
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev.h b/drivers/crypto/cnxk/cnxk_cryptodev.h
index 6760c13..5e38933 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev.h
+++ b/drivers/crypto/cnxk/cnxk_cryptodev.h
@@ -13,7 +13,7 @@
 #define CNXK_CPT_MAX_CAPS	 34
 #define CNXK_SEC_CRYPTO_MAX_CAPS 4
 #define CNXK_SEC_MAX_CAPS	 3
-
+#define CNXK_AE_EC_ID_MAX	 5
 /**
  * Device private data
  */
@@ -23,6 +23,8 @@ struct cnxk_cpt_vf {
 	struct rte_cryptodev_capabilities
 		sec_crypto_caps[CNXK_SEC_CRYPTO_MAX_CAPS];
 	struct rte_security_capability sec_caps[CNXK_SEC_MAX_CAPS];
+	uint64_t cnxk_fpm_iova[CNXK_AE_EC_ID_MAX];
+	struct roc_ae_ec_group *ec_grp[CNXK_AE_EC_ID_MAX];
 };
 
 int cnxk_cpt_eng_grp_add(struct roc_cpt *roc_cpt);
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index 0d81785..7322539 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -8,11 +8,15 @@
 
 #include "roc_cpt.h"
 
+#include "cnxk_ae.h"
 #include "cnxk_cryptodev.h"
 #include "cnxk_cryptodev_ops.h"
 #include "cnxk_cryptodev_capabilities.h"
 #include "cnxk_se.h"
 
+#define CNXK_CPT_MAX_ASYM_OP_NUM_PARAMS 5
+#define CNXK_CPT_MAX_ASYM_OP_MOD_LEN	1024
+
 static int
 cnxk_cpt_get_mlen(void)
 {
@@ -31,6 +35,20 @@ cnxk_cpt_get_mlen(void)
 	return len;
 }
 
+static int
+cnxk_cpt_asym_get_mlen(void)
+{
+	uint32_t len;
+
+	/* To hold RPTR */
+	len = sizeof(uint64_t);
+
+	/* Get meta len for asymmetric operations */
+	len += CNXK_CPT_MAX_ASYM_OP_NUM_PARAMS * CNXK_CPT_MAX_ASYM_OP_MOD_LEN;
+
+	return len;
+}
+
 int
 cnxk_cpt_dev_config(struct rte_cryptodev *dev,
 		    struct rte_cryptodev_config *conf)
@@ -54,6 +72,23 @@ cnxk_cpt_dev_config(struct rte_cryptodev *dev,
 		return ret;
 	}
 
+	if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) {
+		/* Initialize shared FPM table */
+		ret = roc_ae_fpm_get(vf->cnxk_fpm_iova);
+		if (ret) {
+			plt_err("Could not get FPM table");
+			return ret;
+		}
+
+		/* Init EC grp table */
+		ret = roc_ae_ec_grp_get(vf->ec_grp);
+		if (ret) {
+			plt_err("Could not get EC grp table");
+			roc_ae_fpm_put();
+			return ret;
+		}
+	}
+
 	return 0;
 }
 
@@ -86,6 +121,11 @@ cnxk_cpt_dev_close(struct rte_cryptodev *dev)
 		}
 	}
 
+	if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) {
+		roc_ae_fpm_put();
+		roc_ae_ec_grp_put();
+	}
+
 	roc_cpt_dev_clear(&vf->cpt);
 
 	return 0;
@@ -128,6 +168,12 @@ cnxk_cpt_metabuf_mempool_create(const struct rte_cryptodev *dev,
 		mlen = cnxk_cpt_get_mlen();
 	}
 
+	if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) {
+
+		/* Get meta len required for asymmetric operations */
+		mlen = RTE_MAX(mlen, cnxk_cpt_asym_get_mlen());
+	}
+
 	cache_sz = RTE_MIN(RTE_MEMPOOL_CACHE_MAX_SIZE, nb_elements / 1.5);
 
 	/* Allocate mempool */
@@ -549,3 +595,63 @@ cnxk_cpt_sym_session_clear(struct rte_cryptodev *dev,
 {
 	return sym_session_clear(dev->driver_id, sess);
 }
+
+unsigned int
+cnxk_ae_session_size_get(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct cnxk_ae_sess);
+}
+
+void
+cnxk_ae_session_clear(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_asym_session *sess)
+{
+	struct rte_mempool *sess_mp;
+	struct cnxk_ae_sess *priv;
+
+	priv = get_asym_session_private_data(sess, dev->driver_id);
+	if (priv == NULL)
+		return;
+
+	/* Free resources allocated in session_cfg */
+	cnxk_ae_free_session_parameters(priv);
+
+	/* Reset and free object back to pool */
+	memset(priv, 0, cnxk_ae_session_size_get(dev));
+	sess_mp = rte_mempool_from_obj(priv);
+	set_asym_session_private_data(sess, dev->driver_id, NULL);
+	rte_mempool_put(sess_mp, priv);
+}
+
+int
+cnxk_ae_session_cfg(struct rte_cryptodev *dev,
+		    struct rte_crypto_asym_xform *xform,
+		    struct rte_cryptodev_asym_session *sess,
+		    struct rte_mempool *pool)
+{
+	struct cnxk_cpt_vf *vf = dev->data->dev_private;
+	struct roc_cpt *roc_cpt = &vf->cpt;
+	struct cnxk_ae_sess *priv;
+	union cpt_inst_w7 w7;
+	int ret;
+
+	if (rte_mempool_get(pool, (void **)&priv))
+		return -ENOMEM;
+
+	memset(priv, 0, sizeof(struct cnxk_ae_sess));
+
+	ret = cnxk_ae_fill_session_parameters(priv, xform);
+	if (ret) {
+		rte_mempool_put(pool, priv);
+		return ret;
+	}
+
+	w7.u64 = 0;
+	w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_AE];
+	priv->cpt_inst_w7 = w7.u64;
+	priv->cnxk_fpm_iova = vf->cnxk_fpm_iova;
+	priv->ec_grp = vf->ec_grp;
+	set_asym_session_private_data(sess, dev->driver_id, priv);
+
+	return 0;
+}
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
index 7995959..c317f40 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
@@ -105,4 +105,12 @@ void cnxk_cpt_sym_session_clear(struct rte_cryptodev *dev,
 
 void sym_session_clear(int driver_id, struct rte_cryptodev_sym_session *sess);
 
+unsigned int cnxk_ae_session_size_get(struct rte_cryptodev *dev __rte_unused);
+
+void cnxk_ae_session_clear(struct rte_cryptodev *dev,
+			   struct rte_cryptodev_asym_session *sess);
+int cnxk_ae_session_cfg(struct rte_cryptodev *dev,
+			struct rte_crypto_asym_xform *xform,
+			struct rte_cryptodev_asym_session *sess,
+			struct rte_mempool *pool);
 #endif /* _CNXK_CRYPTODEV_OPS_H_ */
-- 
2.7.4