[dpdk-dev v2 2/2] crypto/qat: use intel-ipsec-mb for partial hash & aes

[Kai Ji <kai.ji@intel.com>]

Since openssl 3.0 now deprecates the low level API QAT required to
perform partial hash & aes operation when creating the session. This
patch is to transfer such dependency from openssl to intel-ipsec-mb.

Signed-off-by: Kai Ji <kai.ji@intel.com>
Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
---
 drivers/crypto/qat/qat_sym_session.c | 353 +++++++++++++++++++++++++--
 1 file changed, 334 insertions(+), 19 deletions(-)

diff --git a/drivers/crypto/qat/qat_sym_session.c b/drivers/crypto/qat/qat_sym_session.c
index 9d6a19c0be..3f806a4b55 100644
--- a/drivers/crypto/qat/qat_sym_session.c
+++ b/drivers/crypto/qat/qat_sym_session.c
@@ -7,6 +7,10 @@
 #include <openssl/md5.h>	/* Needed to calculate pre-compute values */
 #include <openssl/evp.h>	/* Needed for bpi runt block processing */
 
+#ifdef RTE_QAT_LIBIPSECMB
+#include <intel-ipsec-mb.h>
+#endif
+
 #include <rte_memcpy.h>
 #include <rte_common.h>
 #include <rte_spinlock.h>
@@ -22,6 +26,10 @@
 #include "qat_sym_session.h"
 #include "qat_sym.h"
 
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+#include <openssl/provider.h>
+#endif
+
 /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
 static const uint8_t sha1InitialState[] = {
 	0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x89, 0x98, 0xba,
@@ -470,6 +478,21 @@ qat_sym_session_configure(struct rte_cryptodev *dev,
 		return -ENOMEM;
 	}
 
+#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+	OSSL_PROVIDER * legacy;
+	OSSL_PROVIDER *deflt;
+
+	/* Load Multiple providers into the default (NULL) library context */
+	legacy = OSSL_PROVIDER_load(NULL, "legacy");
+	if (legacy == NULL)
+		return -EINVAL;
+
+	deflt = OSSL_PROVIDER_load(NULL, "default");
+	if (deflt == NULL) {
+		OSSL_PROVIDER_unload(legacy);
+		return  -EINVAL;
+	}
+#endif
 	ret = qat_sym_session_set_parameters(dev, xform, sess_private_data);
 	if (ret != 0) {
 		QAT_LOG(ERR,
@@ -483,6 +506,10 @@ qat_sym_session_configure(struct rte_cryptodev *dev,
 	set_sym_session_private_data(sess, dev->driver_id,
 		sess_private_data);
 
+# if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
+	OSSL_PROVIDER_unload(legacy);
+	OSSL_PROVIDER_unload(deflt);
+# endif
 	return 0;
 }
 
@@ -1057,6 +1084,297 @@ static int qat_hash_get_block_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 	return -EFAULT;
 }
 
+#define HMAC_IPAD_VALUE	0x36
+#define HMAC_OPAD_VALUE	0x5c
+#define HASH_XCBC_PRECOMP_KEY_NUM 3
+
+static const uint8_t AES_CMAC_SEED[ICP_QAT_HW_AES_128_KEY_SZ];
+
+#ifdef RTE_QAT_LIBIPSECMB
+static int aes_ipsecmb_job(uint8_t *in, uint8_t *out, IMB_MGR *m,
+		const uint8_t *key, uint16_t auth_keylen)
+{
+	int err;
+	struct IMB_JOB *job;
+	DECLARE_ALIGNED(uint32_t expkey[4*15], 16);
+	DECLARE_ALIGNED(uint32_t dust[4*15], 16);
+
+	if (auth_keylen == ICP_QAT_HW_AES_128_KEY_SZ)
+		IMB_AES_KEYEXP_128(m, key, expkey, dust);
+	else if (auth_keylen == ICP_QAT_HW_AES_192_KEY_SZ)
+		IMB_AES_KEYEXP_192(m, key, expkey, dust);
+	else if (auth_keylen == ICP_QAT_HW_AES_256_KEY_SZ)
+		IMB_AES_KEYEXP_256(m, key, expkey, dust);
+	else
+		return -EFAULT;
+
+	job = IMB_GET_NEXT_JOB(m);
+
+	job->src = in;
+	job->dst = out;
+	job->enc_keys = expkey;
+	job->key_len_in_bytes = auth_keylen;
+	job->msg_len_to_cipher_in_bytes = 16;
+	job->iv_len_in_bytes = 0;
+	job->cipher_direction = IMB_DIR_ENCRYPT;
+	job->cipher_mode = IMB_CIPHER_ECB;
+	job->hash_alg = IMB_AUTH_NULL;
+
+	while (IMB_FLUSH_JOB(m) != NULL)
+		;
+
+	job = IMB_SUBMIT_JOB(m);
+	if (job) {
+		if (job->status == IMB_STATUS_COMPLETED)
+			return 0;
+	}
+
+	err = imb_get_errno(m);
+	if (err)
+		QAT_LOG(ERR, "Error: %s!\n", imb_get_strerror(err));
+
+	return -EFAULT;
+}
+
+static int
+partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
+		uint8_t *data_in, uint8_t *data_out)
+{
+	int digest_size;
+	uint8_t digest[qat_hash_get_digest_size(
+			ICP_QAT_HW_AUTH_ALGO_DELIMITER)];
+	uint32_t *hash_state_out_be32;
+	uint64_t *hash_state_out_be64;
+	int i;
+	IMB_MGR *m;
+
+	m = alloc_mb_mgr(0);
+	if (m == NULL)
+		return -ENOMEM;
+	init_mb_mgr_auto(m, NULL);
+
+	/* Initialize to avoid gcc warning */
+	memset(digest, 0, sizeof(digest));
+
+	digest_size = qat_hash_get_digest_size(hash_alg);
+	if (digest_size <= 0)
+		return -EFAULT;
+
+	hash_state_out_be32 = (uint32_t *)data_out;
+	hash_state_out_be64 = (uint64_t *)data_out;
+
+	switch (hash_alg) {
+	case ICP_QAT_HW_AUTH_ALGO_SHA1:
+		IMB_SHA1_ONE_BLOCK(m, data_in, digest);
+		for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++)
+			*hash_state_out_be32 =
+				rte_bswap32(*(((uint32_t *)digest)+i));
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_SHA224:
+		IMB_SHA224_ONE_BLOCK(m, data_in, digest);
+		for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++)
+			*hash_state_out_be32 =
+				rte_bswap32(*(((uint32_t *)digest)+i));
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_SHA256:
+		IMB_SHA256_ONE_BLOCK(m, data_in, digest);
+		for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++)
+			*hash_state_out_be32 =
+				rte_bswap32(*(((uint32_t *)digest)+i));
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_SHA384:
+		IMB_SHA384_ONE_BLOCK(m, data_in, digest);
+		for (i = 0; i < digest_size >> 3; i++, hash_state_out_be64++)
+			*hash_state_out_be64 =
+				rte_bswap64(*(((uint64_t *)digest)+i));
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_SHA512:
+		IMB_SHA512_ONE_BLOCK(m, data_in, digest);
+		for (i = 0; i < digest_size >> 3; i++, hash_state_out_be64++)
+			*hash_state_out_be64 =
+				rte_bswap64(*(((uint64_t *)digest)+i));
+		break;
+	case ICP_QAT_HW_AUTH_ALGO_MD5:
+		IMB_MD5_ONE_BLOCK(m, data_in, data_out);
+		break;
+	default:
+		QAT_LOG(ERR, "invalid hash alg %u", hash_alg);
+		return -EFAULT;
+	}
+	free_mb_mgr(m);
+	return 0;
+}
+
+static int qat_sym_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
+				const uint8_t *auth_key,
+				uint16_t auth_keylen,
+				uint8_t *p_state_buf,
+				uint16_t *p_state_len,
+				uint8_t aes_cmac)
+{
+	int block_size;
+	uint8_t ipad[qat_hash_get_block_size(ICP_QAT_HW_AUTH_ALGO_DELIMITER)];
+	uint8_t opad[qat_hash_get_block_size(ICP_QAT_HW_AUTH_ALGO_DELIMITER)];
+	int i;
+
+	IMB_MGR *m;
+	m = alloc_mb_mgr(0);
+	if (m == NULL)
+		return -ENOMEM;
+
+	init_mb_mgr_auto(m, NULL);
+
+	if (hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC) {
+
+		/* CMAC */
+		if (aes_cmac) {
+			uint8_t *in = NULL;
+			uint8_t k0[ICP_QAT_HW_AES_128_KEY_SZ];
+			uint8_t *k1, *k2;
+
+			auth_keylen = ICP_QAT_HW_AES_128_KEY_SZ;
+
+			in = rte_zmalloc("AES CMAC K1",
+					 ICP_QAT_HW_AES_128_KEY_SZ, 16);
+
+			if (in == NULL) {
+				QAT_LOG(ERR, "Failed to alloc memory");
+				return -ENOMEM;
+			}
+
+			rte_memcpy(in, AES_CMAC_SEED,
+				   ICP_QAT_HW_AES_128_KEY_SZ);
+			rte_memcpy(p_state_buf, auth_key, auth_keylen);
+
+			DECLARE_ALIGNED(uint32_t expkey[4*15], 16);
+			DECLARE_ALIGNED(uint32_t dust[4*15], 16);
+			IMB_AES_KEYEXP_128(m, p_state_buf, expkey, dust);
+			k1 = p_state_buf + ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ;
+			k2 = k1 + ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ;
+
+			IMB_AES_CMAC_SUBKEY_GEN_128(m, expkey, k1, k2);
+			memset(k0, 0, ICP_QAT_HW_AES_128_KEY_SZ);
+			*p_state_len = ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ;
+			rte_free(in);
+			free_mb_mgr(m);
+			return 0;
+		}
+
+		static uint8_t qat_aes_xcbc_key_seed[
+				ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ] = {
+			0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+			0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+			0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+			0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+			0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
+			0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
+		};
+
+		uint8_t *in = NULL;
+		uint8_t *out = p_state_buf;
+		int x;
+
+		in = rte_zmalloc("working mem for key",
+				ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ, 16);
+		if (in == NULL) {
+			QAT_LOG(ERR, "Failed to alloc memory");
+			return -ENOMEM;
+		}
+
+		rte_memcpy(in, qat_aes_xcbc_key_seed,
+				ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ);
+		for (x = 0; x < HASH_XCBC_PRECOMP_KEY_NUM; x++) {
+			if (aes_ipsecmb_job(in, out, m, auth_key, auth_keylen)) {
+				rte_free(in -
+				  (x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ));
+				memset(out -
+				   (x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ),
+				  0, ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ);
+				return -EFAULT;
+			}
+
+			in += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ;
+			out += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ;
+		}
+		*p_state_len = ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ;
+		rte_free(in - x*ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ);
+		free_mb_mgr(m);
+		return 0;
+
+	} else if ((hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) ||
+		(hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64)) {
+		uint8_t *in = NULL;
+		uint8_t *out = p_state_buf;
+
+		memset(p_state_buf, 0, ICP_QAT_HW_GALOIS_H_SZ +
+				ICP_QAT_HW_GALOIS_LEN_A_SZ +
+				ICP_QAT_HW_GALOIS_E_CTR0_SZ);
+		in = rte_zmalloc("working mem for key",
+				ICP_QAT_HW_GALOIS_H_SZ, 16);
+		if (in == NULL) {
+			QAT_LOG(ERR, "Failed to alloc memory");
+			return -ENOMEM;
+		}
+
+		memset(in, 0, ICP_QAT_HW_GALOIS_H_SZ);
+		if (aes_ipsecmb_job(in, out, m, auth_key, auth_keylen))
+			return -EFAULT;
+
+		*p_state_len = ICP_QAT_HW_GALOIS_H_SZ +
+				ICP_QAT_HW_GALOIS_LEN_A_SZ +
+				ICP_QAT_HW_GALOIS_E_CTR0_SZ;
+		rte_free(in);
+		free_mb_mgr(m);
+		return 0;
+	}
+
+	block_size = qat_hash_get_block_size(hash_alg);
+	if (block_size < 0)
+		return block_size;
+	/* init ipad and opad from key and xor with fixed values */
+	memset(ipad, 0, block_size);
+	memset(opad, 0, block_size);
+
+	if (auth_keylen > (unsigned int)block_size) {
+		QAT_LOG(ERR, "invalid keylen %u", auth_keylen);
+		return -EFAULT;
+	}
+	rte_memcpy(ipad, auth_key, auth_keylen);
+	rte_memcpy(opad, auth_key, auth_keylen);
+
+	for (i = 0; i < block_size; i++) {
+		uint8_t *ipad_ptr = ipad + i;
+		uint8_t *opad_ptr = opad + i;
+		*ipad_ptr ^= HMAC_IPAD_VALUE;
+		*opad_ptr ^= HMAC_OPAD_VALUE;
+	}
+
+	/* do partial hash of ipad and copy to state1 */
+	if (partial_hash_compute(hash_alg, ipad, p_state_buf)) {
+		memset(ipad, 0, block_size);
+		memset(opad, 0, block_size);
+		QAT_LOG(ERR, "ipad precompute failed");
+		return -EFAULT;
+	}
+
+	/*
+	 * State len is a multiple of 8, so may be larger than the digest.
+	 * Put the partial hash of opad state_len bytes after state1
+	 */
+	*p_state_len = qat_hash_get_state1_size(hash_alg);
+	if (partial_hash_compute(hash_alg, opad, p_state_buf + *p_state_len)) {
+		memset(ipad, 0, block_size);
+		memset(opad, 0, block_size);
+		QAT_LOG(ERR, "opad precompute failed");
+		return -EFAULT;
+	}
+
+	/*  don't leave data lying around */
+	memset(ipad, 0, block_size);
+	memset(opad, 0, block_size);
+	return 0;
+}
+#else
 static int partial_hash_sha1(uint8_t *data_in, uint8_t *data_out)
 {
 	SHA_CTX ctx;
@@ -1124,6 +1442,20 @@ static int partial_hash_md5(uint8_t *data_in, uint8_t *data_out)
 	return 0;
 }
 
+static void aes_cmac_key_derive(uint8_t *base, uint8_t *derived)
+{
+	int i;
+
+	derived[0] = base[0] << 1;
+	for (i = 1; i < ICP_QAT_HW_AES_BLK_SZ ; i++) {
+		derived[i] = base[i] << 1;
+		derived[i - 1] |= base[i] >> 7;
+	}
+
+	if (base[0] & 0x80)
+		derived[ICP_QAT_HW_AES_BLK_SZ - 1] ^= QAT_AES_CMAC_CONST_RB;
+}
+
 static int
 partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
 		uint8_t *data_in, uint8_t *data_out)
@@ -1192,25 +1524,6 @@ partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
 
 	return 0;
 }
-#define HMAC_IPAD_VALUE	0x36
-#define HMAC_OPAD_VALUE	0x5c
-#define HASH_XCBC_PRECOMP_KEY_NUM 3
-
-static const uint8_t AES_CMAC_SEED[ICP_QAT_HW_AES_128_KEY_SZ];
-
-static void aes_cmac_key_derive(uint8_t *base, uint8_t *derived)
-{
-	int i;
-
-	derived[0] = base[0] << 1;
-	for (i = 1; i < ICP_QAT_HW_AES_BLK_SZ ; i++) {
-		derived[i] = base[i] << 1;
-		derived[i - 1] |= base[i] >> 7;
-	}
-
-	if (base[0] & 0x80)
-		derived[ICP_QAT_HW_AES_BLK_SZ - 1] ^= QAT_AES_CMAC_CONST_RB;
-}
 
 static int qat_sym_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 				const uint8_t *auth_key,
@@ -1279,6 +1592,7 @@ static int qat_sym_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 			uint8_t *in = NULL;
 			uint8_t *out = p_state_buf;
 			int x;
+
 			AES_KEY enc_key;
 
 			in = rte_zmalloc("working mem for key",
@@ -1385,6 +1699,7 @@ static int qat_sym_do_precomputes(enum icp_qat_hw_auth_algo hash_alg,
 	memset(opad, 0, block_size);
 	return 0;
 }
+#endif
 
 static void
 qat_sym_session_init_common_hdr(struct qat_sym_session *session)
-- 
2.17.1