From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by dpdk.org (Postfix) with ESMTP id 89F371B555 for ; Fri, 23 Nov 2018 15:04:17 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 23 Nov 2018 06:04:15 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.56,270,1539673200"; d="scan'208";a="283566317" Received: from silpixa00398673.ir.intel.com (HELO silpixa00398673.ger.corp.intel.com) ([10.237.223.54]) by fmsmga006.fm.intel.com with ESMTP; 23 Nov 2018 06:04:14 -0800 From: Fan Zhang To: dev@dpdk.org Cc: akhil.goyal@nxp.com Date: Fri, 23 Nov 2018 14:04:13 +0000 Message-Id: <20181123140413.52154-1-roy.fan.zhang@intel.com> X-Mailer: git-send-email 2.13.6 Subject: [dpdk-dev] [PATCH] crypto/aesni_mb: use of archtecture independent macros X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 23 Nov 2018 14:04:18 -0000 This patch updates the aesni_mb to use IMB_* arch independent macros to reduce the code size and future maintaining effort. Signed-off-by: Fan Zhang --- This patch targets 19.02 release drivers/crypto/aesni_mb/aesni_mb_ops.h | 302 --------------------- drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c | 160 ++++++----- drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 15 +- drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h | 33 ++- 4 files changed, 118 insertions(+), 392 deletions(-) delete mode 100644 drivers/crypto/aesni_mb/aesni_mb_ops.h diff --git a/drivers/crypto/aesni_mb/aesni_mb_ops.h b/drivers/crypto/aesni_mb/aesni_mb_ops.h deleted file mode 100644 index 575d6a5b8..000000000 --- a/drivers/crypto/aesni_mb/aesni_mb_ops.h +++ /dev/null @@ -1,302 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2015 Intel Corporation - */ - -#ifndef _AESNI_MB_OPS_H_ -#define _AESNI_MB_OPS_H_ - -#ifndef LINUX -#define LINUX -#endif - -#include - -/* - * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50, - * so if macro is not defined, it means that the version is 0.49. - */ -#if !defined(IMB_VERSION_NUM) -#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c)) -#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0) -#endif - -enum aesni_mb_vector_mode { - RTE_AESNI_MB_NOT_SUPPORTED = 0, - RTE_AESNI_MB_SSE, - RTE_AESNI_MB_AVX, - RTE_AESNI_MB_AVX2, - RTE_AESNI_MB_AVX512 -}; - -typedef void (*md5_one_block_t)(const void *data, void *digest); - -typedef void (*sha1_one_block_t)(const void *data, void *digest); -typedef void (*sha224_one_block_t)(const void *data, void *digest); -typedef void (*sha256_one_block_t)(const void *data, void *digest); -typedef void (*sha384_one_block_t)(const void *data, void *digest); -typedef void (*sha512_one_block_t)(const void *data, void *digest); - -typedef void (*aes_keyexp_128_t) - (const void *key, void *enc_exp_keys, void *dec_exp_keys); -typedef void (*aes_keyexp_192_t) - (const void *key, void *enc_exp_keys, void *dec_exp_keys); -typedef void (*aes_keyexp_256_t) - (const void *key, void *enc_exp_keys, void *dec_exp_keys); -typedef void (*aes_xcbc_expand_key_t) - (const void *key, void *exp_k1, void *k2, void *k3); -typedef void (*aes_cmac_sub_key_gen_t) - (const void *exp_key, void *k2, void *k3); -typedef void (*aes_cmac_keyexp_t) - (const void *key, void *keyexp); -typedef void (*aes_gcm_keyexp_t) - (const void *key, struct gcm_key_data *keyexp); - -/** Multi-buffer library function pointer table */ -struct aesni_mb_op_fns { - struct { - init_mb_mgr_t init_mgr; - /**< Initialise scheduler */ - get_next_job_t get_next; - /**< Get next free job structure */ - submit_job_t submit; - /**< Submit job to scheduler */ - get_completed_job_t get_completed_job; - /**< Get completed job */ - flush_job_t flush_job; - /**< flush jobs from manager */ - } job; - /**< multi buffer manager functions */ - - struct { - struct { - md5_one_block_t md5; - /**< MD5 one block hash */ - sha1_one_block_t sha1; - /**< SHA1 one block hash */ - sha224_one_block_t sha224; - /**< SHA224 one block hash */ - sha256_one_block_t sha256; - /**< SHA256 one block hash */ - sha384_one_block_t sha384; - /**< SHA384 one block hash */ - sha512_one_block_t sha512; - /**< SHA512 one block hash */ - } one_block; - /**< one block hash functions */ - - struct { - aes_keyexp_128_t aes128; - /**< AES128 key expansions */ - aes_keyexp_192_t aes192; - /**< AES192 key expansions */ - aes_keyexp_256_t aes256; - /**< AES256 key expansions */ - aes_xcbc_expand_key_t aes_xcbc; - /**< AES XCBC key epansions */ - aes_cmac_sub_key_gen_t aes_cmac_subkey; - /**< AES CMAC subkey expansions */ - aes_cmac_keyexp_t aes_cmac_expkey; - /**< AES CMAC key expansions */ - aes_gcm_keyexp_t aes_gcm_128; - /**< AES GCM 128 key expansions */ - aes_gcm_keyexp_t aes_gcm_192; - /**< AES GCM 192 key expansions */ - aes_gcm_keyexp_t aes_gcm_256; - /**< AES GCM 256 key expansions */ - } keyexp; - /**< Key expansion functions */ -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) - struct { - hash_fn_t sha1; - hash_fn_t sha224; - hash_fn_t sha256; - hash_fn_t sha384; - hash_fn_t sha512; - } multi_block; - /** multi block hash functions */ -#endif - } aux; - /**< Auxiliary functions */ -}; - - -static const struct aesni_mb_op_fns job_ops[] = { - [RTE_AESNI_MB_NOT_SUPPORTED] = { - .job = { - NULL - }, - .aux = { - .one_block = { - NULL - }, - .keyexp = { - NULL - }, -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) - .multi_block = { - NULL - } -#endif - - } - }, - [RTE_AESNI_MB_SSE] = { - .job = { - init_mb_mgr_sse, - get_next_job_sse, - submit_job_sse, - get_completed_job_sse, - flush_job_sse - }, - .aux = { - .one_block = { - md5_one_block_sse, - sha1_one_block_sse, - sha224_one_block_sse, - sha256_one_block_sse, - sha384_one_block_sse, - sha512_one_block_sse - }, - .keyexp = { - aes_keyexp_128_sse, - aes_keyexp_192_sse, - aes_keyexp_256_sse, - aes_xcbc_expand_key_sse, - aes_cmac_subkey_gen_sse, - aes_keyexp_128_enc_sse, - aes_gcm_pre_128_sse, - aes_gcm_pre_192_sse, - aes_gcm_pre_256_sse - }, -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) - .multi_block = { - sha1_sse, - sha224_sse, - sha256_sse, - sha384_sse, - sha512_sse - } -#endif - } - }, - [RTE_AESNI_MB_AVX] = { - .job = { - init_mb_mgr_avx, - get_next_job_avx, - submit_job_avx, - get_completed_job_avx, - flush_job_avx - }, - .aux = { - .one_block = { - md5_one_block_avx, - sha1_one_block_avx, - sha224_one_block_avx, - sha256_one_block_avx, - sha384_one_block_avx, - sha512_one_block_avx - }, - .keyexp = { - aes_keyexp_128_avx, - aes_keyexp_192_avx, - aes_keyexp_256_avx, - aes_xcbc_expand_key_avx, - aes_cmac_subkey_gen_avx, - aes_keyexp_128_enc_avx, - aes_gcm_pre_128_avx_gen2, - aes_gcm_pre_192_avx_gen2, - aes_gcm_pre_256_avx_gen2 - }, -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) - .multi_block = { - sha1_avx, - sha224_avx, - sha256_avx, - sha384_avx, - sha512_avx - } -#endif - } - }, - [RTE_AESNI_MB_AVX2] = { - .job = { - init_mb_mgr_avx2, - get_next_job_avx2, - submit_job_avx2, - get_completed_job_avx2, - flush_job_avx2 - }, - .aux = { - .one_block = { - md5_one_block_avx2, - sha1_one_block_avx2, - sha224_one_block_avx2, - sha256_one_block_avx2, - sha384_one_block_avx2, - sha512_one_block_avx2 - }, - .keyexp = { - aes_keyexp_128_avx2, - aes_keyexp_192_avx2, - aes_keyexp_256_avx2, - aes_xcbc_expand_key_avx2, - aes_cmac_subkey_gen_avx2, - aes_keyexp_128_enc_avx2, - aes_gcm_pre_128_avx_gen4, - aes_gcm_pre_192_avx_gen4, - aes_gcm_pre_256_avx_gen4 - }, -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) - .multi_block = { - sha1_avx2, - sha224_avx2, - sha256_avx2, - sha384_avx2, - sha512_avx2 - } -#endif - } - }, - [RTE_AESNI_MB_AVX512] = { - .job = { - init_mb_mgr_avx512, - get_next_job_avx512, - submit_job_avx512, - get_completed_job_avx512, - flush_job_avx512 - }, - .aux = { - .one_block = { - md5_one_block_avx512, - sha1_one_block_avx512, - sha224_one_block_avx512, - sha256_one_block_avx512, - sha384_one_block_avx512, - sha512_one_block_avx512 - }, - .keyexp = { - aes_keyexp_128_avx512, - aes_keyexp_192_avx512, - aes_keyexp_256_avx512, - aes_xcbc_expand_key_avx512, - aes_cmac_subkey_gen_avx512, - aes_keyexp_128_enc_avx512, - aes_gcm_pre_128_avx_gen4, - aes_gcm_pre_192_avx_gen4, - aes_gcm_pre_256_avx_gen4 - }, -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) - .multi_block = { - sha1_avx512, - sha224_avx512, - sha256_avx512, - sha384_avx512, - sha512_avx512 - } -#endif - } - } -}; - - -#endif /* _AESNI_MB_OPS_H_ */ diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 83250e32c..0f593b2a3 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -100,7 +100,7 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform) /** Set session authentication parameters */ static int -aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, +aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { @@ -135,13 +135,16 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, return -EINVAL; } sess->auth.gen_digest_len = sess->auth.req_digest_len; - (*mb_ops->aux.keyexp.aes_xcbc)(xform->auth.key.data, + + IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data, sess->auth.xcbc.k1_expanded, sess->auth.xcbc.k2, sess->auth.xcbc.k3); return 0; } if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) { + uint32_t dust[4*15]; + sess->auth.algo = AES_CMAC; uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC); @@ -169,10 +172,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, sess->auth.gen_digest_len = cmac_digest_len; else sess->auth.gen_digest_len = sess->auth.req_digest_len; - (*mb_ops->aux.keyexp.aes_cmac_expkey)(xform->auth.key.data, - sess->auth.cmac.expkey); - (*mb_ops->aux.keyexp.aes_cmac_subkey)(sess->auth.cmac.expkey, + IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data, + sess->auth.cmac.expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey, sess->auth.cmac.skey1, sess->auth.cmac.skey2); return 0; } @@ -180,14 +183,14 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_MD5_HMAC: sess->auth.algo = MD5; - hash_oneblock_fn = mb_ops->aux.one_block.md5; + hash_oneblock_fn = mb_mgr->md5_one_block; break; case RTE_CRYPTO_AUTH_SHA1_HMAC: sess->auth.algo = SHA1; - hash_oneblock_fn = mb_ops->aux.one_block.sha1; -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + hash_oneblock_fn = mb_mgr->sha1_one_block; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0) if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) { - mb_ops->aux.multi_block.sha1( + IMB_SHA1(mb_mgr, xform->auth.key.data, xform->auth.key.length, hashed_key); @@ -197,10 +200,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, break; case RTE_CRYPTO_AUTH_SHA224_HMAC: sess->auth.algo = SHA_224; - hash_oneblock_fn = mb_ops->aux.one_block.sha224; -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + hash_oneblock_fn = mb_mgr->sha224_one_block; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0) if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) { - mb_ops->aux.multi_block.sha224( + IMB_SHA224(mb_mgr, xform->auth.key.data, xform->auth.key.length, hashed_key); @@ -210,10 +213,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, break; case RTE_CRYPTO_AUTH_SHA256_HMAC: sess->auth.algo = SHA_256; - hash_oneblock_fn = mb_ops->aux.one_block.sha256; -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + hash_oneblock_fn = mb_mgr->sha256_one_block; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0) if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) { - mb_ops->aux.multi_block.sha256( + IMB_SHA256(mb_mgr, xform->auth.key.data, xform->auth.key.length, hashed_key); @@ -223,10 +226,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, break; case RTE_CRYPTO_AUTH_SHA384_HMAC: sess->auth.algo = SHA_384; - hash_oneblock_fn = mb_ops->aux.one_block.sha384; -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + hash_oneblock_fn = mb_mgr->sha384_one_block; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0) if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) { - mb_ops->aux.multi_block.sha384( + IMB_SHA384(mb_mgr, xform->auth.key.data, xform->auth.key.length, hashed_key); @@ -236,10 +239,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, break; case RTE_CRYPTO_AUTH_SHA512_HMAC: sess->auth.algo = SHA_512; - hash_oneblock_fn = mb_ops->aux.one_block.sha512; -#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) + hash_oneblock_fn = mb_mgr->sha512_one_block; +#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0) if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) { - mb_ops->aux.multi_block.sha512( + IMB_SHA512(mb_mgr, xform->auth.key.data, xform->auth.key.length, hashed_key); @@ -292,13 +295,12 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops, /** Set session cipher parameters */ static int -aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, +aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { uint8_t is_aes = 0; uint8_t is_3DES = 0; - aes_keyexp_t aes_keyexp_fn; if (xform == NULL) { sess->cipher.mode = NULL_CIPHER; @@ -361,26 +363,26 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, switch (xform->cipher.key.length) { case AES_128_BYTES: sess->cipher.key_length_in_bytes = AES_128_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes128; + IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); break; case AES_192_BYTES: sess->cipher.key_length_in_bytes = AES_192_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes192; + IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); break; case AES_256_BYTES: sess->cipher.key_length_in_bytes = AES_256_BYTES; - aes_keyexp_fn = mb_ops->aux.keyexp.aes256; + IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); break; default: AESNI_MB_LOG(ERR, "Invalid cipher key length"); return -EINVAL; } - - /* Expanded cipher keys */ - (*aes_keyexp_fn)(xform->cipher.key.data, - sess->cipher.expanded_aes_keys.encode, - sess->cipher.expanded_aes_keys.decode); - } else if (is_3DES) { uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0], sess->cipher.exp_3des_keys.key[1], @@ -388,9 +390,12 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, switch (xform->cipher.key.length) { case 24: - des_key_schedule(keys[0], xform->cipher.key.data); - des_key_schedule(keys[1], xform->cipher.key.data+8); - des_key_schedule(keys[2], xform->cipher.key.data+16); + IMB_DES_KEYSCHED(mb_mgr, keys[0], + xform->cipher.key.data); + IMB_DES_KEYSCHED(mb_mgr, keys[1], + xform->cipher.key.data + 8); + IMB_DES_KEYSCHED(mb_mgr, keys[2], + xform->cipher.key.data + 16); /* Initialize keys - 24 bytes: [K1-K2-K3] */ sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0]; @@ -398,16 +403,18 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2]; break; case 16: - des_key_schedule(keys[0], xform->cipher.key.data); - des_key_schedule(keys[1], xform->cipher.key.data+8); - + IMB_DES_KEYSCHED(mb_mgr, keys[0], + xform->cipher.key.data); + IMB_DES_KEYSCHED(mb_mgr, keys[1], + xform->cipher.key.data + 8); /* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */ sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0]; sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1]; sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0]; break; case 8: - des_key_schedule(keys[0], xform->cipher.key.data); + IMB_DES_KEYSCHED(mb_mgr, keys[0], + xform->cipher.key.data); /* Initialize keys - 8 bytes: [K1 = K2 = K3] */ sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0]; @@ -431,9 +438,11 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, } sess->cipher.key_length_in_bytes = 8; - des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.encode, + IMB_DES_KEYSCHED(mb_mgr, + (uint64_t *)sess->cipher.expanded_aes_keys.encode, xform->cipher.key.data); - des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.decode, + IMB_DES_KEYSCHED(mb_mgr, + (uint64_t *)sess->cipher.expanded_aes_keys.decode, xform->cipher.key.data); } @@ -441,15 +450,10 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops, } static int -aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, +aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { - union { - aes_keyexp_t aes_keyexp_fn; - aes_gcm_keyexp_t aes_gcm_keyexp_fn; - } keyexp; - switch (xform->aead.op) { case RTE_CRYPTO_AEAD_OP_ENCRYPT: sess->cipher.direction = ENCRYPT; @@ -473,17 +477,15 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, switch (xform->aead.key.length) { case AES_128_BYTES: sess->cipher.key_length_in_bytes = AES_128_BYTES; - keyexp.aes_keyexp_fn = mb_ops->aux.keyexp.aes128; + IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); break; default: AESNI_MB_LOG(ERR, "Invalid cipher key length"); return -EINVAL; } - /* Expanded cipher keys */ - (*keyexp.aes_keyexp_fn)(xform->aead.key.data, - sess->cipher.expanded_aes_keys.encode, - sess->cipher.expanded_aes_keys.decode); break; case RTE_CRYPTO_AEAD_AES_GCM: @@ -493,26 +495,24 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, switch (xform->aead.key.length) { case AES_128_BYTES: sess->cipher.key_length_in_bytes = AES_128_BYTES; - keyexp.aes_gcm_keyexp_fn = - mb_ops->aux.keyexp.aes_gcm_128; + IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data, + &sess->cipher.gcm_key); break; case AES_192_BYTES: sess->cipher.key_length_in_bytes = AES_192_BYTES; - keyexp.aes_gcm_keyexp_fn = - mb_ops->aux.keyexp.aes_gcm_192; + IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data, + &sess->cipher.gcm_key); break; case AES_256_BYTES: sess->cipher.key_length_in_bytes = AES_256_BYTES; - keyexp.aes_gcm_keyexp_fn = - mb_ops->aux.keyexp.aes_gcm_256; + IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data, + &sess->cipher.gcm_key); break; default: AESNI_MB_LOG(ERR, "Invalid cipher key length"); return -EINVAL; } - (keyexp.aes_gcm_keyexp_fn)(xform->aead.key.data, - &sess->cipher.gcm_key); break; default: @@ -539,7 +539,7 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops, /** Parse crypto xform chain and set private session parameters */ int -aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, +aesni_mb_set_session_parameters(const MB_MGR *mb_mgr, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { @@ -598,13 +598,13 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, /* Default IV length = 0 */ sess->iv.length = 0; - ret = aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform); + ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform); if (ret != 0) { AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters"); return ret; } - ret = aesni_mb_set_session_cipher_parameters(mb_ops, sess, + ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess, cipher_xform); if (ret != 0) { AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters"); @@ -612,7 +612,7 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, } if (aead_xform) { - ret = aesni_mb_set_session_aead_parameters(mb_ops, sess, + ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess, aead_xform); if (ret != 0) { AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters"); @@ -673,7 +673,7 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) sess = (struct aesni_mb_session *)_sess_private_data; - if (unlikely(aesni_mb_set_session_parameters(qp->op_fns, + if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr, sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); rte_mempool_put(qp->sess_mp, _sess_private_data); @@ -989,7 +989,7 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job, if (processed_jobs == nb_ops) break; - job = (*qp->op_fns->job.get_completed_job)(qp->mb_mgr); + job = IMB_GET_COMPLETED_JOB(qp->mb_mgr); } return processed_jobs; @@ -1002,7 +1002,7 @@ flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops, int processed_ops = 0; /* Flush the remaining jobs */ - JOB_AES_HMAC *job = (*qp->op_fns->job.flush_job)(qp->mb_mgr); + JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr); if (job) processed_ops += handle_completed_jobs(qp, job, @@ -1042,7 +1042,7 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, uint8_t digest_idx = qp->digest_idx; do { /* Get next free mb job struct from mb manager */ - job = (*qp->op_fns->job.get_next)(qp->mb_mgr); + job = IMB_GET_NEXT_JOB(qp->mb_mgr); if (unlikely(job == NULL)) { /* if no free mb job structs we need to flush mb_mgr */ processed_jobs += flush_mb_mgr(qp, @@ -1052,7 +1052,7 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, if (nb_ops == processed_jobs) break; - job = (*qp->op_fns->job.get_next)(qp->mb_mgr); + job = IMB_GET_NEXT_JOB(qp->mb_mgr); } /* @@ -1072,8 +1072,11 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, } /* Submit job to multi-buffer for processing */ - job = (*qp->op_fns->job.submit)(qp->mb_mgr); - +#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG + job = IMB_SUBMIT_JOB(qp->mb_mgr); +#else + job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr); +#endif /* * If submit returns a processed job then handle it, * before submitting subsequent jobs @@ -1105,6 +1108,7 @@ cryptodev_aesni_mb_create(const char *name, struct rte_cryptodev *dev; struct aesni_mb_private *internals; enum aesni_mb_vector_mode vector_mode; + MB_MGR *mb_mgr; /* Check CPU for support for AES instruction set */ if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { @@ -1139,18 +1143,26 @@ cryptodev_aesni_mb_create(const char *name, RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | RTE_CRYPTODEV_FF_CPU_AESNI; + mb_mgr = alloc_mb_mgr(0); + if (mb_mgr == NULL) + return -ENOMEM; + switch (vector_mode) { case RTE_AESNI_MB_SSE: dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + init_mb_mgr_sse(mb_mgr); break; case RTE_AESNI_MB_AVX: dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + init_mb_mgr_avx(mb_mgr); break; case RTE_AESNI_MB_AVX2: dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx2(mb_mgr); break; case RTE_AESNI_MB_AVX512: dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512; + init_mb_mgr_avx512(mb_mgr); break; default: break; @@ -1161,6 +1173,7 @@ cryptodev_aesni_mb_create(const char *name, internals->vector_mode = vector_mode; internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; + internals->mb_mgr = mb_mgr; #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0) AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n", @@ -1204,6 +1217,7 @@ static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev) { struct rte_cryptodev *cryptodev; + struct aesni_mb_private *internals; const char *name; name = rte_vdev_device_name(vdev); @@ -1214,6 +1228,10 @@ cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev) if (cryptodev == NULL) return -ENODEV; + internals = cryptodev->data->dev_private; + + free_mb_mgr(internals->mb_mgr); + return rte_cryptodev_pmd_destroy(cryptodev); } diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index f3eff2685..bfbd97923 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -509,8 +509,6 @@ aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) r = rte_ring_lookup(qp->name); if (r) rte_ring_free(r); - if (qp->mb_mgr) - free_mb_mgr(qp->mb_mgr); rte_free(qp); dev->data->queue_pairs[qp_id] = NULL; } @@ -589,13 +587,7 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, goto qp_setup_cleanup; - qp->mb_mgr = alloc_mb_mgr(0); - if (qp->mb_mgr == NULL) { - ret = -ENOMEM; - goto qp_setup_cleanup; - } - - qp->op_fns = &job_ops[internals->vector_mode]; + qp->mb_mgr = internals->mb_mgr; qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp, qp_conf->nb_descriptors, socket_id); @@ -612,9 +604,6 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, "digest_mp_%u_%u", dev->data->dev_id, qp_id); - - /* Initialise multi-buffer manager */ - (*qp->op_fns->job.init_mgr)(qp->mb_mgr); return 0; qp_setup_cleanup: @@ -663,7 +652,7 @@ aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev, return -ENOMEM; } - ret = aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], + ret = aesni_mb_set_session_parameters(internals->mb_mgr, sess_private_data, xform); if (ret != 0) { AESNI_MB_LOG(ERR, "failed configure session parameters"); diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h index d8021cdaa..1dc2c5bf0 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h @@ -5,7 +5,24 @@ #ifndef _RTE_AESNI_MB_PMD_PRIVATE_H_ #define _RTE_AESNI_MB_PMD_PRIVATE_H_ -#include "aesni_mb_ops.h" +#include + +/* + * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50, + * so if macro is not defined, it means that the version is 0.49. + */ +#if !defined(IMB_VERSION_NUM) +#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c)) +#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0) +#endif + +enum aesni_mb_vector_mode { + RTE_AESNI_MB_NOT_SUPPORTED = 0, + RTE_AESNI_MB_SSE, + RTE_AESNI_MB_AVX, + RTE_AESNI_MB_AVX2, + RTE_AESNI_MB_AVX512 +}; #define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb /**< AES-NI Multi buffer PMD device name */ @@ -83,7 +100,9 @@ static const unsigned auth_digest_byte_lengths[] = { [AES_XCBC] = 16, [AES_CMAC] = 16, [AES_GMAC] = 12, - [NULL_HASH] = 0 + [NULL_HASH] = 0, + /**< Vector mode dependent pointer table of the multi-buffer APIs */ + }; /** @@ -115,6 +134,8 @@ struct aesni_mb_private { /**< CPU vector instruction set mode */ unsigned max_nb_queue_pairs; /**< Max number of queue pairs supported by device */ + MB_MGR *mb_mgr; + /**< Multi-buffer instance */ }; /** AESNI Multi buffer queue pair */ @@ -123,8 +144,6 @@ struct aesni_mb_qp { /**< Queue Pair Identifier */ char name[RTE_CRYPTODEV_NAME_MAX_LEN]; /**< Unique Queue Pair Name */ - const struct aesni_mb_op_fns *op_fns; - /**< Vector mode dependent pointer table of the multi-buffer APIs */ MB_MGR *mb_mgr; /**< Multi-buffer instance */ struct rte_ring *ingress_queue; @@ -153,7 +172,9 @@ struct aesni_mb_session { } iv; /**< IV parameters */ - /** Cipher Parameters */ + /** Cipher Parameters */const struct aesni_mb_op_fns *op_fns; + /**< Vector mode dependent pointer table of the multi-buffer APIs */ + struct { /** Cipher direction - encrypt / decrypt */ JOB_CIPHER_DIRECTION direction; @@ -238,7 +259,7 @@ struct aesni_mb_session { * */ extern int -aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops, +aesni_mb_set_session_parameters(const MB_MGR *mb_mgr, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform); -- 2.13.6