[dpdk-dev] [PATCH] crypto/aesni_mb: use of archtecture independent macros

[Fan Zhang <roy.fan.zhang@intel.com>]

This patch updates the aesni_mb to use IMB_* arch independent
macros to reduce the code size and future maintaining effort.

Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
---
This patch targets 19.02 release

 drivers/crypto/aesni_mb/aesni_mb_ops.h             | 302 ---------------------
 drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c         | 160 ++++++-----
 drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c     |  15 +-
 drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h |  33 ++-
 4 files changed, 118 insertions(+), 392 deletions(-)
 delete mode 100644 drivers/crypto/aesni_mb/aesni_mb_ops.h

diff --git a/drivers/crypto/aesni_mb/aesni_mb_ops.h b/drivers/crypto/aesni_mb/aesni_mb_ops.h
deleted file mode 100644
index 575d6a5b8..000000000
--- a/drivers/crypto/aesni_mb/aesni_mb_ops.h
+++ /dev/null
@@ -1,302 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015 Intel Corporation
- */
-
-#ifndef _AESNI_MB_OPS_H_
-#define _AESNI_MB_OPS_H_
-
-#ifndef LINUX
-#define LINUX
-#endif
-
-#include <intel-ipsec-mb.h>
-
-/*
- * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
- * so if macro is not defined, it means that the version is 0.49.
- */
-#if !defined(IMB_VERSION_NUM)
-#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
-#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
-#endif
-
-enum aesni_mb_vector_mode {
-	RTE_AESNI_MB_NOT_SUPPORTED = 0,
-	RTE_AESNI_MB_SSE,
-	RTE_AESNI_MB_AVX,
-	RTE_AESNI_MB_AVX2,
-	RTE_AESNI_MB_AVX512
-};
-
-typedef void (*md5_one_block_t)(const void *data, void *digest);
-
-typedef void (*sha1_one_block_t)(const void *data, void *digest);
-typedef void (*sha224_one_block_t)(const void *data, void *digest);
-typedef void (*sha256_one_block_t)(const void *data, void *digest);
-typedef void (*sha384_one_block_t)(const void *data, void *digest);
-typedef void (*sha512_one_block_t)(const void *data, void *digest);
-
-typedef void (*aes_keyexp_128_t)
-		(const void *key, void *enc_exp_keys, void *dec_exp_keys);
-typedef void (*aes_keyexp_192_t)
-		(const void *key, void *enc_exp_keys, void *dec_exp_keys);
-typedef void (*aes_keyexp_256_t)
-		(const void *key, void *enc_exp_keys, void *dec_exp_keys);
-typedef void (*aes_xcbc_expand_key_t)
-		(const void *key, void *exp_k1, void *k2, void *k3);
-typedef void (*aes_cmac_sub_key_gen_t)
-		(const void *exp_key, void *k2, void *k3);
-typedef void (*aes_cmac_keyexp_t)
-		(const void *key, void *keyexp);
-typedef void (*aes_gcm_keyexp_t)
-		(const void *key, struct gcm_key_data *keyexp);
-
-/** Multi-buffer library function pointer table */
-struct aesni_mb_op_fns {
-	struct {
-		init_mb_mgr_t init_mgr;
-		/**< Initialise scheduler  */
-		get_next_job_t get_next;
-		/**< Get next free job structure */
-		submit_job_t submit;
-		/**< Submit job to scheduler */
-		get_completed_job_t get_completed_job;
-		/**< Get completed job */
-		flush_job_t flush_job;
-		/**< flush jobs from manager */
-	} job;
-	/**< multi buffer manager functions */
-
-	struct {
-		struct {
-			md5_one_block_t md5;
-			/**< MD5 one block hash */
-			sha1_one_block_t sha1;
-			/**< SHA1 one block hash */
-			sha224_one_block_t sha224;
-			/**< SHA224 one block hash */
-			sha256_one_block_t sha256;
-			/**< SHA256 one block hash */
-			sha384_one_block_t sha384;
-			/**< SHA384 one block hash */
-			sha512_one_block_t sha512;
-			/**< SHA512 one block hash */
-		} one_block;
-		/**< one block hash functions */
-
-		struct {
-			aes_keyexp_128_t aes128;
-			/**< AES128 key expansions */
-			aes_keyexp_192_t aes192;
-			/**< AES192 key expansions */
-			aes_keyexp_256_t aes256;
-			/**< AES256 key expansions */
-			aes_xcbc_expand_key_t aes_xcbc;
-			/**< AES XCBC key epansions */
-			aes_cmac_sub_key_gen_t aes_cmac_subkey;
-			/**< AES CMAC subkey expansions */
-			aes_cmac_keyexp_t aes_cmac_expkey;
-			/**< AES CMAC key expansions */
-			aes_gcm_keyexp_t aes_gcm_128;
-			/**< AES GCM 128 key expansions */
-			aes_gcm_keyexp_t aes_gcm_192;
-			/**< AES GCM 192 key expansions */
-			aes_gcm_keyexp_t aes_gcm_256;
-			/**< AES GCM 256 key expansions */
-		} keyexp;
-		/**< Key expansion functions */
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
-		struct {
-			hash_fn_t sha1;
-			hash_fn_t sha224;
-			hash_fn_t sha256;
-			hash_fn_t sha384;
-			hash_fn_t sha512;
-		} multi_block;
-		/** multi block hash functions */
-#endif
-	} aux;
-	/**< Auxiliary functions */
-};
-
-
-static const struct aesni_mb_op_fns job_ops[] = {
-		[RTE_AESNI_MB_NOT_SUPPORTED] = {
-			.job = {
-				NULL
-			},
-			.aux = {
-				.one_block = {
-					NULL
-				},
-				.keyexp = {
-					NULL
-				},
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
-				.multi_block = {
-					NULL
-				}
-#endif
-
-			}
-		},
-		[RTE_AESNI_MB_SSE] = {
-			.job = {
-				init_mb_mgr_sse,
-				get_next_job_sse,
-				submit_job_sse,
-				get_completed_job_sse,
-				flush_job_sse
-			},
-			.aux = {
-				.one_block = {
-					md5_one_block_sse,
-					sha1_one_block_sse,
-					sha224_one_block_sse,
-					sha256_one_block_sse,
-					sha384_one_block_sse,
-					sha512_one_block_sse
-				},
-				.keyexp = {
-					aes_keyexp_128_sse,
-					aes_keyexp_192_sse,
-					aes_keyexp_256_sse,
-					aes_xcbc_expand_key_sse,
-					aes_cmac_subkey_gen_sse,
-					aes_keyexp_128_enc_sse,
-					aes_gcm_pre_128_sse,
-					aes_gcm_pre_192_sse,
-					aes_gcm_pre_256_sse
-				},
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
-				.multi_block = {
-					sha1_sse,
-					sha224_sse,
-					sha256_sse,
-					sha384_sse,
-					sha512_sse
-				}
-#endif
-			}
-		},
-		[RTE_AESNI_MB_AVX] = {
-			.job = {
-				init_mb_mgr_avx,
-				get_next_job_avx,
-				submit_job_avx,
-				get_completed_job_avx,
-				flush_job_avx
-			},
-			.aux = {
-				.one_block = {
-					md5_one_block_avx,
-					sha1_one_block_avx,
-					sha224_one_block_avx,
-					sha256_one_block_avx,
-					sha384_one_block_avx,
-					sha512_one_block_avx
-				},
-				.keyexp = {
-					aes_keyexp_128_avx,
-					aes_keyexp_192_avx,
-					aes_keyexp_256_avx,
-					aes_xcbc_expand_key_avx,
-					aes_cmac_subkey_gen_avx,
-					aes_keyexp_128_enc_avx,
-					aes_gcm_pre_128_avx_gen2,
-					aes_gcm_pre_192_avx_gen2,
-					aes_gcm_pre_256_avx_gen2
-				},
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
-				.multi_block = {
-					sha1_avx,
-					sha224_avx,
-					sha256_avx,
-					sha384_avx,
-					sha512_avx
-				}
-#endif
-			}
-		},
-		[RTE_AESNI_MB_AVX2] = {
-			.job = {
-				init_mb_mgr_avx2,
-				get_next_job_avx2,
-				submit_job_avx2,
-				get_completed_job_avx2,
-				flush_job_avx2
-			},
-			.aux = {
-				.one_block = {
-					md5_one_block_avx2,
-					sha1_one_block_avx2,
-					sha224_one_block_avx2,
-					sha256_one_block_avx2,
-					sha384_one_block_avx2,
-					sha512_one_block_avx2
-				},
-				.keyexp = {
-					aes_keyexp_128_avx2,
-					aes_keyexp_192_avx2,
-					aes_keyexp_256_avx2,
-					aes_xcbc_expand_key_avx2,
-					aes_cmac_subkey_gen_avx2,
-					aes_keyexp_128_enc_avx2,
-					aes_gcm_pre_128_avx_gen4,
-					aes_gcm_pre_192_avx_gen4,
-					aes_gcm_pre_256_avx_gen4
-				},
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
-				.multi_block = {
-					sha1_avx2,
-					sha224_avx2,
-					sha256_avx2,
-					sha384_avx2,
-					sha512_avx2
-				}
-#endif
-			}
-		},
-		[RTE_AESNI_MB_AVX512] = {
-			.job = {
-				init_mb_mgr_avx512,
-				get_next_job_avx512,
-				submit_job_avx512,
-				get_completed_job_avx512,
-				flush_job_avx512
-			},
-			.aux = {
-				.one_block = {
-					md5_one_block_avx512,
-					sha1_one_block_avx512,
-					sha224_one_block_avx512,
-					sha256_one_block_avx512,
-					sha384_one_block_avx512,
-					sha512_one_block_avx512
-				},
-				.keyexp = {
-					aes_keyexp_128_avx512,
-					aes_keyexp_192_avx512,
-					aes_keyexp_256_avx512,
-					aes_xcbc_expand_key_avx512,
-					aes_cmac_subkey_gen_avx512,
-					aes_keyexp_128_enc_avx512,
-					aes_gcm_pre_128_avx_gen4,
-					aes_gcm_pre_192_avx_gen4,
-					aes_gcm_pre_256_avx_gen4
-				},
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
-				.multi_block = {
-					sha1_avx512,
-					sha224_avx512,
-					sha256_avx512,
-					sha384_avx512,
-					sha512_avx512
-				}
-#endif
-			}
-		}
-};
-
-
-#endif /* _AESNI_MB_OPS_H_ */
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
index 83250e32c..0f593b2a3 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
@@ -100,7 +100,7 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
 
 /** Set session authentication parameters */
 static int
-aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
 		struct aesni_mb_session *sess,
 		const struct rte_crypto_sym_xform *xform)
 {
@@ -135,13 +135,16 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 			return -EINVAL;
 		}
 		sess->auth.gen_digest_len = sess->auth.req_digest_len;
-		(*mb_ops->aux.keyexp.aes_xcbc)(xform->auth.key.data,
+
+		IMB_AES_XCBC_KEYEXP(mb_mgr, xform->auth.key.data,
 				sess->auth.xcbc.k1_expanded,
 				sess->auth.xcbc.k2, sess->auth.xcbc.k3);
 		return 0;
 	}
 
 	if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_CMAC) {
+		uint32_t dust[4*15];
+
 		sess->auth.algo = AES_CMAC;
 
 		uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
@@ -169,10 +172,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 			sess->auth.gen_digest_len = cmac_digest_len;
 		else
 			sess->auth.gen_digest_len = sess->auth.req_digest_len;
-		(*mb_ops->aux.keyexp.aes_cmac_expkey)(xform->auth.key.data,
-				sess->auth.cmac.expkey);
 
-		(*mb_ops->aux.keyexp.aes_cmac_subkey)(sess->auth.cmac.expkey,
+		IMB_AES_KEYEXP_128(mb_mgr, xform->auth.key.data,
+				sess->auth.cmac.expkey, dust);
+		IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, sess->auth.cmac.expkey,
 				sess->auth.cmac.skey1, sess->auth.cmac.skey2);
 		return 0;
 	}
@@ -180,14 +183,14 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 	switch (xform->auth.algo) {
 	case RTE_CRYPTO_AUTH_MD5_HMAC:
 		sess->auth.algo = MD5;
-		hash_oneblock_fn = mb_ops->aux.one_block.md5;
+		hash_oneblock_fn = mb_mgr->md5_one_block;
 		break;
 	case RTE_CRYPTO_AUTH_SHA1_HMAC:
 		sess->auth.algo = SHA1;
-		hash_oneblock_fn = mb_ops->aux.one_block.sha1;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		hash_oneblock_fn = mb_mgr->sha1_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
 		if (xform->auth.key.length > get_auth_algo_blocksize(SHA1)) {
-			mb_ops->aux.multi_block.sha1(
+			IMB_SHA1(mb_mgr,
 				xform->auth.key.data,
 				xform->auth.key.length,
 				hashed_key);
@@ -197,10 +200,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 		break;
 	case RTE_CRYPTO_AUTH_SHA224_HMAC:
 		sess->auth.algo = SHA_224;
-		hash_oneblock_fn = mb_ops->aux.one_block.sha224;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		hash_oneblock_fn = mb_mgr->sha224_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
 		if (xform->auth.key.length > get_auth_algo_blocksize(SHA_224)) {
-			mb_ops->aux.multi_block.sha224(
+			IMB_SHA224(mb_mgr,
 				xform->auth.key.data,
 				xform->auth.key.length,
 				hashed_key);
@@ -210,10 +213,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 		break;
 	case RTE_CRYPTO_AUTH_SHA256_HMAC:
 		sess->auth.algo = SHA_256;
-		hash_oneblock_fn = mb_ops->aux.one_block.sha256;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		hash_oneblock_fn = mb_mgr->sha256_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
 		if (xform->auth.key.length > get_auth_algo_blocksize(SHA_256)) {
-			mb_ops->aux.multi_block.sha256(
+			IMB_SHA256(mb_mgr,
 				xform->auth.key.data,
 				xform->auth.key.length,
 				hashed_key);
@@ -223,10 +226,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 		break;
 	case RTE_CRYPTO_AUTH_SHA384_HMAC:
 		sess->auth.algo = SHA_384;
-		hash_oneblock_fn = mb_ops->aux.one_block.sha384;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		hash_oneblock_fn = mb_mgr->sha384_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
 		if (xform->auth.key.length > get_auth_algo_blocksize(SHA_384)) {
-			mb_ops->aux.multi_block.sha384(
+			IMB_SHA384(mb_mgr,
 				xform->auth.key.data,
 				xform->auth.key.length,
 				hashed_key);
@@ -236,10 +239,10 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 		break;
 	case RTE_CRYPTO_AUTH_SHA512_HMAC:
 		sess->auth.algo = SHA_512;
-		hash_oneblock_fn = mb_ops->aux.one_block.sha512;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
+		hash_oneblock_fn = mb_mgr->sha512_one_block;
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 52, 0)
 		if (xform->auth.key.length > get_auth_algo_blocksize(SHA_512)) {
-			mb_ops->aux.multi_block.sha512(
+			IMB_SHA512(mb_mgr,
 				xform->auth.key.data,
 				xform->auth.key.length,
 				hashed_key);
@@ -292,13 +295,12 @@ aesni_mb_set_session_auth_parameters(const struct aesni_mb_op_fns *mb_ops,
 
 /** Set session cipher parameters */
 static int
-aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
 		struct aesni_mb_session *sess,
 		const struct rte_crypto_sym_xform *xform)
 {
 	uint8_t is_aes = 0;
 	uint8_t is_3DES = 0;
-	aes_keyexp_t aes_keyexp_fn;
 
 	if (xform == NULL) {
 		sess->cipher.mode = NULL_CIPHER;
@@ -361,26 +363,26 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 		switch (xform->cipher.key.length) {
 		case AES_128_BYTES:
 			sess->cipher.key_length_in_bytes = AES_128_BYTES;
-			aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
+			IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
+					sess->cipher.expanded_aes_keys.encode,
+					sess->cipher.expanded_aes_keys.decode);
 			break;
 		case AES_192_BYTES:
 			sess->cipher.key_length_in_bytes = AES_192_BYTES;
-			aes_keyexp_fn = mb_ops->aux.keyexp.aes192;
+			IMB_AES_KEYEXP_192(mb_mgr, xform->cipher.key.data,
+					sess->cipher.expanded_aes_keys.encode,
+					sess->cipher.expanded_aes_keys.decode);
 			break;
 		case AES_256_BYTES:
 			sess->cipher.key_length_in_bytes = AES_256_BYTES;
-			aes_keyexp_fn = mb_ops->aux.keyexp.aes256;
+			IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
+					sess->cipher.expanded_aes_keys.encode,
+					sess->cipher.expanded_aes_keys.decode);
 			break;
 		default:
 			AESNI_MB_LOG(ERR, "Invalid cipher key length");
 			return -EINVAL;
 		}
-
-		/* Expanded cipher keys */
-		(*aes_keyexp_fn)(xform->cipher.key.data,
-				sess->cipher.expanded_aes_keys.encode,
-				sess->cipher.expanded_aes_keys.decode);
-
 	} else if (is_3DES) {
 		uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0],
 				sess->cipher.exp_3des_keys.key[1],
@@ -388,9 +390,12 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 
 		switch (xform->cipher.key.length) {
 		case  24:
-			des_key_schedule(keys[0], xform->cipher.key.data);
-			des_key_schedule(keys[1], xform->cipher.key.data+8);
-			des_key_schedule(keys[2], xform->cipher.key.data+16);
+			IMB_DES_KEYSCHED(mb_mgr, keys[0],
+					xform->cipher.key.data);
+			IMB_DES_KEYSCHED(mb_mgr, keys[1],
+					xform->cipher.key.data + 8);
+			IMB_DES_KEYSCHED(mb_mgr, keys[2],
+					xform->cipher.key.data + 16);
 
 			/* Initialize keys - 24 bytes: [K1-K2-K3] */
 			sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
@@ -398,16 +403,18 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 			sess->cipher.exp_3des_keys.ks_ptr[2] = keys[2];
 			break;
 		case 16:
-			des_key_schedule(keys[0], xform->cipher.key.data);
-			des_key_schedule(keys[1], xform->cipher.key.data+8);
-
+			IMB_DES_KEYSCHED(mb_mgr, keys[0],
+					xform->cipher.key.data);
+			IMB_DES_KEYSCHED(mb_mgr, keys[1],
+					xform->cipher.key.data + 8);
 			/* Initialize keys - 16 bytes: [K1=K1,K2=K2,K3=K1] */
 			sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
 			sess->cipher.exp_3des_keys.ks_ptr[1] = keys[1];
 			sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
 			break;
 		case 8:
-			des_key_schedule(keys[0], xform->cipher.key.data);
+			IMB_DES_KEYSCHED(mb_mgr, keys[0],
+					xform->cipher.key.data);
 
 			/* Initialize keys - 8 bytes: [K1 = K2 = K3] */
 			sess->cipher.exp_3des_keys.ks_ptr[0] = keys[0];
@@ -431,9 +438,11 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 		}
 		sess->cipher.key_length_in_bytes = 8;
 
-		des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.encode,
+		IMB_DES_KEYSCHED(mb_mgr,
+			(uint64_t *)sess->cipher.expanded_aes_keys.encode,
 				xform->cipher.key.data);
-		des_key_schedule((uint64_t *)sess->cipher.expanded_aes_keys.decode,
+		IMB_DES_KEYSCHED(mb_mgr,
+			(uint64_t *)sess->cipher.expanded_aes_keys.decode,
 				xform->cipher.key.data);
 	}
 
@@ -441,15 +450,10 @@ aesni_mb_set_session_cipher_parameters(const struct aesni_mb_op_fns *mb_ops,
 }
 
 static int
-aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
 		struct aesni_mb_session *sess,
 		const struct rte_crypto_sym_xform *xform)
 {
-	union {
-		aes_keyexp_t aes_keyexp_fn;
-		aes_gcm_keyexp_t aes_gcm_keyexp_fn;
-	} keyexp;
-
 	switch (xform->aead.op) {
 	case RTE_CRYPTO_AEAD_OP_ENCRYPT:
 		sess->cipher.direction = ENCRYPT;
@@ -473,17 +477,15 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 		switch (xform->aead.key.length) {
 		case AES_128_BYTES:
 			sess->cipher.key_length_in_bytes = AES_128_BYTES;
-			keyexp.aes_keyexp_fn = mb_ops->aux.keyexp.aes128;
+			IMB_AES_KEYEXP_128(mb_mgr, xform->aead.key.data,
+					sess->cipher.expanded_aes_keys.encode,
+					sess->cipher.expanded_aes_keys.decode);
 			break;
 		default:
 			AESNI_MB_LOG(ERR, "Invalid cipher key length");
 			return -EINVAL;
 		}
 
-		/* Expanded cipher keys */
-		(*keyexp.aes_keyexp_fn)(xform->aead.key.data,
-				sess->cipher.expanded_aes_keys.encode,
-				sess->cipher.expanded_aes_keys.decode);
 		break;
 
 	case RTE_CRYPTO_AEAD_AES_GCM:
@@ -493,26 +495,24 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 		switch (xform->aead.key.length) {
 		case AES_128_BYTES:
 			sess->cipher.key_length_in_bytes = AES_128_BYTES;
-			keyexp.aes_gcm_keyexp_fn =
-					mb_ops->aux.keyexp.aes_gcm_128;
+			IMB_AES128_GCM_PRE(mb_mgr, xform->aead.key.data,
+				&sess->cipher.gcm_key);
 			break;
 		case AES_192_BYTES:
 			sess->cipher.key_length_in_bytes = AES_192_BYTES;
-			keyexp.aes_gcm_keyexp_fn =
-					mb_ops->aux.keyexp.aes_gcm_192;
+			IMB_AES192_GCM_PRE(mb_mgr, xform->aead.key.data,
+				&sess->cipher.gcm_key);
 			break;
 		case AES_256_BYTES:
 			sess->cipher.key_length_in_bytes = AES_256_BYTES;
-			keyexp.aes_gcm_keyexp_fn =
-					mb_ops->aux.keyexp.aes_gcm_256;
+			IMB_AES256_GCM_PRE(mb_mgr, xform->aead.key.data,
+				&sess->cipher.gcm_key);
 			break;
 		default:
 			AESNI_MB_LOG(ERR, "Invalid cipher key length");
 			return -EINVAL;
 		}
 
-		(keyexp.aes_gcm_keyexp_fn)(xform->aead.key.data,
-				&sess->cipher.gcm_key);
 		break;
 
 	default:
@@ -539,7 +539,7 @@ aesni_mb_set_session_aead_parameters(const struct aesni_mb_op_fns *mb_ops,
 
 /** Parse crypto xform chain and set private session parameters */
 int
-aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 		struct aesni_mb_session *sess,
 		const struct rte_crypto_sym_xform *xform)
 {
@@ -598,13 +598,13 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
 	/* Default IV length = 0 */
 	sess->iv.length = 0;
 
-	ret = aesni_mb_set_session_auth_parameters(mb_ops, sess, auth_xform);
+	ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
 	if (ret != 0) {
 		AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
 		return ret;
 	}
 
-	ret = aesni_mb_set_session_cipher_parameters(mb_ops, sess,
+	ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
 			cipher_xform);
 	if (ret != 0) {
 		AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
@@ -612,7 +612,7 @@ aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
 	}
 
 	if (aead_xform) {
-		ret = aesni_mb_set_session_aead_parameters(mb_ops, sess,
+		ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
 				aead_xform);
 		if (ret != 0) {
 			AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
@@ -673,7 +673,7 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
 
 		sess = (struct aesni_mb_session *)_sess_private_data;
 
-		if (unlikely(aesni_mb_set_session_parameters(qp->op_fns,
+		if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
 				sess, op->sym->xform) != 0)) {
 			rte_mempool_put(qp->sess_mp, _sess);
 			rte_mempool_put(qp->sess_mp, _sess_private_data);
@@ -989,7 +989,7 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
 		if (processed_jobs == nb_ops)
 			break;
 
-		job = (*qp->op_fns->job.get_completed_job)(qp->mb_mgr);
+		job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
 	}
 
 	return processed_jobs;
@@ -1002,7 +1002,7 @@ flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
 	int processed_ops = 0;
 
 	/* Flush the remaining jobs */
-	JOB_AES_HMAC *job = (*qp->op_fns->job.flush_job)(qp->mb_mgr);
+	JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
 
 	if (job)
 		processed_ops += handle_completed_jobs(qp, job,
@@ -1042,7 +1042,7 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	uint8_t digest_idx = qp->digest_idx;
 	do {
 		/* Get next free mb job struct from mb manager */
-		job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
+		job = IMB_GET_NEXT_JOB(qp->mb_mgr);
 		if (unlikely(job == NULL)) {
 			/* if no free mb job structs we need to flush mb_mgr */
 			processed_jobs += flush_mb_mgr(qp,
@@ -1052,7 +1052,7 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 			if (nb_ops == processed_jobs)
 				break;
 
-			job = (*qp->op_fns->job.get_next)(qp->mb_mgr);
+			job = IMB_GET_NEXT_JOB(qp->mb_mgr);
 		}
 
 		/*
@@ -1072,8 +1072,11 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		}
 
 		/* Submit job to multi-buffer for processing */
-		job = (*qp->op_fns->job.submit)(qp->mb_mgr);
-
+#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
+		job = IMB_SUBMIT_JOB(qp->mb_mgr);
+#else
+		job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
+#endif
 		/*
 		 * If submit returns a processed job then handle it,
 		 * before submitting subsequent jobs
@@ -1105,6 +1108,7 @@ cryptodev_aesni_mb_create(const char *name,
 	struct rte_cryptodev *dev;
 	struct aesni_mb_private *internals;
 	enum aesni_mb_vector_mode vector_mode;
+	MB_MGR *mb_mgr;
 
 	/* Check CPU for support for AES instruction set */
 	if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
@@ -1139,18 +1143,26 @@ cryptodev_aesni_mb_create(const char *name,
 			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 			RTE_CRYPTODEV_FF_CPU_AESNI;
 
+	mb_mgr = alloc_mb_mgr(0);
+	if (mb_mgr == NULL)
+		return -ENOMEM;
+
 	switch (vector_mode) {
 	case RTE_AESNI_MB_SSE:
 		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
+		init_mb_mgr_sse(mb_mgr);
 		break;
 	case RTE_AESNI_MB_AVX:
 		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
+		init_mb_mgr_avx(mb_mgr);
 		break;
 	case RTE_AESNI_MB_AVX2:
 		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
+		init_mb_mgr_avx2(mb_mgr);
 		break;
 	case RTE_AESNI_MB_AVX512:
 		dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
+		init_mb_mgr_avx512(mb_mgr);
 		break;
 	default:
 		break;
@@ -1161,6 +1173,7 @@ cryptodev_aesni_mb_create(const char *name,
 
 	internals->vector_mode = vector_mode;
 	internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
+	internals->mb_mgr = mb_mgr;
 
 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 	AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
@@ -1204,6 +1217,7 @@ static int
 cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
 {
 	struct rte_cryptodev *cryptodev;
+	struct aesni_mb_private *internals;
 	const char *name;
 
 	name = rte_vdev_device_name(vdev);
@@ -1214,6 +1228,10 @@ cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
 	if (cryptodev == NULL)
 		return -ENODEV;
 
+	internals = cryptodev->data->dev_private;
+
+	free_mb_mgr(internals->mb_mgr);
+
 	return rte_cryptodev_pmd_destroy(cryptodev);
 }
 
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
index f3eff2685..bfbd97923 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
@@ -509,8 +509,6 @@ aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		r = rte_ring_lookup(qp->name);
 		if (r)
 			rte_ring_free(r);
-		if (qp->mb_mgr)
-			free_mb_mgr(qp->mb_mgr);
 		rte_free(qp);
 		dev->data->queue_pairs[qp_id] = NULL;
 	}
@@ -589,13 +587,7 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		goto qp_setup_cleanup;
 
 
-	qp->mb_mgr = alloc_mb_mgr(0);
-	if (qp->mb_mgr == NULL) {
-		ret = -ENOMEM;
-		goto qp_setup_cleanup;
-	}
-
-	qp->op_fns = &job_ops[internals->vector_mode];
+	qp->mb_mgr = internals->mb_mgr;
 
 	qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
 			qp_conf->nb_descriptors, socket_id);
@@ -612,9 +604,6 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
 				"digest_mp_%u_%u", dev->data->dev_id, qp_id);
-
-	/* Initialise multi-buffer manager */
-	(*qp->op_fns->job.init_mgr)(qp->mb_mgr);
 	return 0;
 
 qp_setup_cleanup:
@@ -663,7 +652,7 @@ aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
 		return -ENOMEM;
 	}
 
-	ret = aesni_mb_set_session_parameters(&job_ops[internals->vector_mode],
+	ret = aesni_mb_set_session_parameters(internals->mb_mgr,
 			sess_private_data, xform);
 	if (ret != 0) {
 		AESNI_MB_LOG(ERR, "failed configure session parameters");
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
index d8021cdaa..1dc2c5bf0 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h
@@ -5,7 +5,24 @@
 #ifndef _RTE_AESNI_MB_PMD_PRIVATE_H_
 #define _RTE_AESNI_MB_PMD_PRIVATE_H_
 
-#include "aesni_mb_ops.h"
+#include <intel-ipsec-mb.h>
+
+/*
+ * IMB_VERSION_NUM macro was introduced in version Multi-buffer 0.50,
+ * so if macro is not defined, it means that the version is 0.49.
+ */
+#if !defined(IMB_VERSION_NUM)
+#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
+#define IMB_VERSION_NUM IMB_VERSION(0, 49, 0)
+#endif
+
+enum aesni_mb_vector_mode {
+	RTE_AESNI_MB_NOT_SUPPORTED = 0,
+	RTE_AESNI_MB_SSE,
+	RTE_AESNI_MB_AVX,
+	RTE_AESNI_MB_AVX2,
+	RTE_AESNI_MB_AVX512
+};
 
 #define CRYPTODEV_NAME_AESNI_MB_PMD	crypto_aesni_mb
 /**< AES-NI Multi buffer PMD device name */
@@ -83,7 +100,9 @@ static const unsigned auth_digest_byte_lengths[] = {
 		[AES_XCBC]	= 16,
 		[AES_CMAC]	= 16,
 		[AES_GMAC]	= 12,
-		[NULL_HASH]		= 0
+		[NULL_HASH]	= 0,
+	/**< Vector mode dependent pointer table of the multi-buffer APIs */
+
 };
 
 /**
@@ -115,6 +134,8 @@ struct aesni_mb_private {
 	/**< CPU vector instruction set mode */
 	unsigned max_nb_queue_pairs;
 	/**< Max number of queue pairs supported by device */
+	MB_MGR *mb_mgr;
+	/**< Multi-buffer instance */
 };
 
 /** AESNI Multi buffer queue pair */
@@ -123,8 +144,6 @@ struct aesni_mb_qp {
 	/**< Queue Pair Identifier */
 	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 	/**< Unique Queue Pair Name */
-	const struct aesni_mb_op_fns *op_fns;
-	/**< Vector mode dependent pointer table of the multi-buffer APIs */
 	MB_MGR *mb_mgr;
 	/**< Multi-buffer instance */
 	struct rte_ring *ingress_queue;
@@ -153,7 +172,9 @@ struct aesni_mb_session {
 	} iv;
 	/**< IV parameters */
 
-	/** Cipher Parameters */
+	/** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;
+	/**< Vector mode dependent pointer table of the multi-buffer APIs */
+
 	struct {
 		/** Cipher direction - encrypt / decrypt */
 		JOB_CIPHER_DIRECTION direction;
@@ -238,7 +259,7 @@ struct aesni_mb_session {
  *
  */
 extern int
-aesni_mb_set_session_parameters(const struct aesni_mb_op_fns *mb_ops,
+aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 		struct aesni_mb_session *sess,
 		const struct rte_crypto_sym_xform *xform);
 
-- 
2.13.6