RE: [PATCH v4] crypto/ipsec_mb: unified IPsec MB interface

[Wathsala Wathawana Vithanage <wathsala.vithanage@arm.com>]

> Subject: [PATCH v4] crypto/ipsec_mb: unified IPsec MB interface
> 
> Currently IPsec MB provides both the JOB API and direct API.
> AESNI_MB PMD is using the JOB API codepath while ZUC, KASUMI, SNOW3G
> and CHACHA20_POLY1305 are using the direct API.
> Instead of using the direct API for these PMDs, they should now make
> use of the JOB API codepath. This would remove all use of the IPsec MB
> direct API for these PMDs.
> 
> Signed-off-by: Brian Dooley <brian.dooley@intel.com>

Acked-by: Wathsala Vithanage <wathsala.vithanage@arm.com>

> ---
> v2:
> - Fix compilation failure
> v3:
> - Remove session configure pointer for each PMD
> v4:
> - Keep AES GCM PMD and fix extern issue
> ---
>  doc/guides/rel_notes/release_24_03.rst        |   6 +
>  drivers/crypto/ipsec_mb/pmd_aesni_mb.c        |  10 +-
>  drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h   |  15 +-
>  drivers/crypto/ipsec_mb/pmd_chacha_poly.c     | 338 +----------
>  .../crypto/ipsec_mb/pmd_chacha_poly_priv.h    |  28 -
>  drivers/crypto/ipsec_mb/pmd_kasumi.c          | 410 +------------
>  drivers/crypto/ipsec_mb/pmd_kasumi_priv.h     |  20 -
>  drivers/crypto/ipsec_mb/pmd_snow3g.c          | 543 +-----------------
>  drivers/crypto/ipsec_mb/pmd_snow3g_priv.h     |  21 -
>  drivers/crypto/ipsec_mb/pmd_zuc.c             | 347 +----------
>  drivers/crypto/ipsec_mb/pmd_zuc_priv.h        |  20 -
>  11 files changed, 48 insertions(+), 1710 deletions(-)
> 
> diff --git a/doc/guides/rel_notes/release_24_03.rst
> b/doc/guides/rel_notes/release_24_03.rst
> index 879bb4944c..6c5b76cef5 100644
> --- a/doc/guides/rel_notes/release_24_03.rst
> +++ b/doc/guides/rel_notes/release_24_03.rst
> @@ -138,6 +138,12 @@ New Features
>      to support TLS v1.2, TLS v1.3 and DTLS v1.2.
>    * Added PMD API to allow raw submission of instructions to CPT.
> 
> +* **Updated ipsec_mb crypto driver.**
> +
> +  * Kasumi, Snow3G, ChaChaPoly and ZUC PMDs now share the job API
> codepath
> +    with AESNI_MB PMD. Depending on the architecture, the performance of
> ZUC
> +    crypto PMD is approximately 10% less for small fixed packet sizes.
> +
> 
>  Removed Items
>  -------------
> diff --git a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
> b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
> index 4de4866cf3..7d4dbc91ef 100644
> --- a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
> +++ b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
> @@ -8,6 +8,8 @@
> 
>  RTE_DEFINE_PER_LCORE(pid_t, pid);
> 
> +uint8_t pmd_driver_id_aesni_mb;
> +
>  struct aesni_mb_op_buf_data {
>  	struct rte_mbuf *m;
>  	uint32_t offset;
> @@ -761,7 +763,7 @@ aesni_mb_set_session_aead_parameters(const
> IMB_MGR *mb_mgr,
>  }
> 
>  /** Configure a aesni multi-buffer session from a crypto xform chain */
> -static int
> +int
>  aesni_mb_session_configure(IMB_MGR *mb_mgr,
>  		void *priv_sess,
>  		const struct rte_crypto_sym_xform *xform)
> @@ -2131,7 +2133,7 @@ set_job_null_op(IMB_JOB *job, struct
> rte_crypto_op *op)
>  }
> 
>  #if IMB_VERSION(1, 2, 0) < IMB_VERSION_NUM
> -static uint16_t
> +uint16_t
>  aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
>  		uint16_t nb_ops)
>  {
> @@ -2321,7 +2323,7 @@ flush_mb_mgr(struct ipsec_mb_qp *qp, IMB_MGR
> *mb_mgr,
>  	return processed_ops;
>  }
> 
> -static uint16_t
> +uint16_t
>  aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
>  		uint16_t nb_ops)
>  {
> @@ -2456,7 +2458,7 @@ verify_sync_dgst(struct rte_crypto_sym_vec *vec,
>  	return k;
>  }
> 
> -static uint32_t
> +uint32_t
>  aesni_mb_process_bulk(struct rte_cryptodev *dev __rte_unused,
>  	struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs
> sofs,
>  	struct rte_crypto_sym_vec *vec)
> diff --git a/drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h
> b/drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h
> index 85994fe5a1..2d462a7f68 100644
> --- a/drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h
> +++ b/drivers/crypto/ipsec_mb/pmd_aesni_mb_priv.h
> @@ -21,6 +21,19 @@
>  #define MAX_NUM_SEGS 16
>  #endif
> 
> +int
> +aesni_mb_session_configure(IMB_MGR * m __rte_unused, void *priv_sess,
> +		const struct rte_crypto_sym_xform *xform);
> +
> +uint16_t
> +aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
> +		uint16_t nb_ops);
> +
> +uint32_t
> +aesni_mb_process_bulk(struct rte_cryptodev *dev __rte_unused,
> +	struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs
> sofs,
> +	struct rte_crypto_sym_vec *vec);
> +
>  static const struct rte_cryptodev_capabilities aesni_mb_capabilities[] = {
>  	{	/* MD5 HMAC */
>  		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
> @@ -722,8 +735,6 @@ static const struct rte_cryptodev_capabilities
> aesni_mb_capabilities[] = {
>  	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
>  };
> 
> -uint8_t pmd_driver_id_aesni_mb;
> -
>  struct aesni_mb_qp_data {
>  	uint8_t temp_digests[IMB_MAX_JOBS][DIGEST_LENGTH_MAX];
>  	/* *< Buffers used to store the digest generated
> diff --git a/drivers/crypto/ipsec_mb/pmd_chacha_poly.c
> b/drivers/crypto/ipsec_mb/pmd_chacha_poly.c
> index 97e7cef233..7436353fc2 100644
> --- a/drivers/crypto/ipsec_mb/pmd_chacha_poly.c
> +++ b/drivers/crypto/ipsec_mb/pmd_chacha_poly.c
> @@ -3,334 +3,7 @@
>   */
> 
>  #include "pmd_chacha_poly_priv.h"
> -
> -/** Parse crypto xform chain and set private session parameters. */
> -static int
> -chacha20_poly1305_session_configure(IMB_MGR * mb_mgr __rte_unused,
> -		void *priv_sess, const struct rte_crypto_sym_xform *xform)
> -{
> -	struct chacha20_poly1305_session *sess = priv_sess;
> -	const struct rte_crypto_sym_xform *auth_xform;
> -	const struct rte_crypto_sym_xform *cipher_xform;
> -	const struct rte_crypto_sym_xform *aead_xform;
> -
> -	uint8_t key_length;
> -	const uint8_t *key;
> -	enum ipsec_mb_operation mode;
> -	int ret = 0;
> -
> -	ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
> -				&cipher_xform, &aead_xform);
> -	if (ret)
> -		return ret;
> -
> -	sess->op = mode;
> -
> -	switch (sess->op) {
> -	case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
> -	case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
> -		if (aead_xform->aead.algo !=
> -				RTE_CRYPTO_AEAD_CHACHA20_POLY1305) {
> -			IPSEC_MB_LOG(ERR,
> -			"The only combined operation supported is
> CHACHA20 POLY1305");
> -			ret = -ENOTSUP;
> -			goto error_exit;
> -		}
> -		/* Set IV parameters */
> -		sess->iv.offset = aead_xform->aead.iv.offset;
> -		sess->iv.length = aead_xform->aead.iv.length;
> -		key_length = aead_xform->aead.key.length;
> -		key = aead_xform->aead.key.data;
> -		sess->aad_length = aead_xform->aead.aad_length;
> -		sess->req_digest_length = aead_xform->aead.digest_length;
> -		break;
> -	default:
> -		IPSEC_MB_LOG(
> -		    ERR, "Wrong xform type, has to be AEAD or
> authentication");
> -		ret = -ENOTSUP;
> -		goto error_exit;
> -	}
> -
> -	/* IV check */
> -	if (sess->iv.length != CHACHA20_POLY1305_IV_LENGTH &&
> -		sess->iv.length != 0) {
> -		IPSEC_MB_LOG(ERR, "Wrong IV length");
> -		ret = -EINVAL;
> -		goto error_exit;
> -	}
> -
> -	/* Check key length */
> -	if (key_length != CHACHA20_POLY1305_KEY_SIZE) {
> -		IPSEC_MB_LOG(ERR, "Invalid key length");
> -		ret = -EINVAL;
> -		goto error_exit;
> -	} else {
> -		memcpy(sess->key, key, CHACHA20_POLY1305_KEY_SIZE);
> -	}
> -
> -	/* Digest check */
> -	if (sess->req_digest_length !=
> CHACHA20_POLY1305_DIGEST_LENGTH) {
> -		IPSEC_MB_LOG(ERR, "Invalid digest length");
> -		ret = -EINVAL;
> -		goto error_exit;
> -	} else {
> -		sess->gen_digest_length =
> CHACHA20_POLY1305_DIGEST_LENGTH;
> -	}
> -
> -error_exit:
> -	return ret;
> -}
> -
> -/**
> - * Process a crypto operation, calling
> - * the direct chacha poly API from the multi buffer library.
> - *
> - * @param	qp		queue pair
> - * @param	op		symmetric crypto operation
> - * @param	session		chacha poly session
> - *
> - * @return
> - * - Return 0 if success
> - */
> -static int
> -chacha20_poly1305_crypto_op(struct ipsec_mb_qp *qp, struct
> rte_crypto_op *op,
> -		struct chacha20_poly1305_session *session)
> -{
> -	struct chacha20_poly1305_qp_data *qp_data =
> -					ipsec_mb_get_qp_private_data(qp);
> -	uint8_t *src, *dst;
> -	uint8_t *iv_ptr;
> -	struct rte_crypto_sym_op *sym_op = op->sym;
> -	struct rte_mbuf *m_src = sym_op->m_src;
> -	uint32_t offset, data_offset, data_length;
> -	uint32_t part_len, data_len;
> -	int total_len;
> -	uint8_t *tag;
> -	unsigned int oop = 0;
> -
> -	offset = sym_op->aead.data.offset;
> -	data_offset = offset;
> -	data_length = sym_op->aead.data.length;
> -	RTE_ASSERT(m_src != NULL);
> -
> -	while (offset >= m_src->data_len && data_length != 0) {
> -		offset -= m_src->data_len;
> -		m_src = m_src->next;
> -
> -		RTE_ASSERT(m_src != NULL);
> -	}
> -
> -	src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset);
> -
> -	data_len = m_src->data_len - offset;
> -	part_len = (data_len < data_length) ? data_len :
> -			data_length;
> -
> -	/* In-place */
> -	if (sym_op->m_dst == NULL || (sym_op->m_dst == sym_op->m_src))
> -		dst = src;
> -	/* Out-of-place */
> -	else {
> -		oop = 1;
> -		/* Segmented destination buffer is not supported
> -		 * if operation is Out-of-place
> -		 */
> -		RTE_ASSERT(rte_pktmbuf_is_contiguous(sym_op->m_dst));
> -		dst = rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *,
> -					data_offset);
> -	}
> -
> -	iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
> -				session->iv.offset);
> -
> -	IMB_CHACHA20_POLY1305_INIT(qp->mb_mgr, session->key,
> -				&qp_data->chacha20_poly1305_ctx_data,
> -				iv_ptr,	sym_op->aead.aad.data,
> -				(uint64_t)session->aad_length);
> -
> -	if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT) {
> -		IMB_CHACHA20_POLY1305_ENC_UPDATE(qp->mb_mgr,
> -				session->key,
> -				&qp_data->chacha20_poly1305_ctx_data,
> -				dst, src, (uint64_t)part_len);
> -		total_len = data_length - part_len;
> -
> -		while (total_len) {
> -			m_src = m_src->next;
> -			RTE_ASSERT(m_src != NULL);
> -
> -			src = rte_pktmbuf_mtod(m_src, uint8_t *);
> -			if (oop)
> -				dst += part_len;
> -			else
> -				dst = src;
> -			part_len = (m_src->data_len < total_len) ?
> -					m_src->data_len : total_len;
> -
> -			if (dst == NULL || src == NULL) {
> -				IPSEC_MB_LOG(ERR, "Invalid src or dst
> input");
> -				return -EINVAL;
> -			}
> -			IMB_CHACHA20_POLY1305_ENC_UPDATE(qp-
> >mb_mgr,
> -					session->key,
> -					&qp_data-
> >chacha20_poly1305_ctx_data,
> -					dst, src, (uint64_t)part_len);
> -			total_len -= part_len;
> -			if (total_len < 0) {
> -				IPSEC_MB_LOG(ERR, "Invalid part len");
> -				return -EINVAL;
> -			}
> -		}
> -
> -		tag = sym_op->aead.digest.data;
> -		IMB_CHACHA20_POLY1305_ENC_FINALIZE(qp->mb_mgr,
> -					&qp_data-
> >chacha20_poly1305_ctx_data,
> -					tag, session->gen_digest_length);
> -
> -	} else {
> -		IMB_CHACHA20_POLY1305_DEC_UPDATE(qp->mb_mgr,
> -					session->key,
> -					&qp_data-
> >chacha20_poly1305_ctx_data,
> -					dst, src, (uint64_t)part_len);
> -
> -		total_len = data_length - part_len;
> -
> -		while (total_len) {
> -			m_src = m_src->next;
> -
> -			RTE_ASSERT(m_src != NULL);
> -
> -			src = rte_pktmbuf_mtod(m_src, uint8_t *);
> -			if (oop)
> -				dst += part_len;
> -			else
> -				dst = src;
> -			part_len = (m_src->data_len < total_len) ?
> -					m_src->data_len : total_len;
> -
> -			if (dst == NULL || src == NULL) {
> -				IPSEC_MB_LOG(ERR, "Invalid src or dst
> input");
> -				return -EINVAL;
> -			}
> -			IMB_CHACHA20_POLY1305_DEC_UPDATE(qp-
> >mb_mgr,
> -					session->key,
> -					&qp_data-
> >chacha20_poly1305_ctx_data,
> -					dst, src, (uint64_t)part_len);
> -			total_len -= part_len;
> -			if (total_len < 0) {
> -				IPSEC_MB_LOG(ERR, "Invalid part len");
> -				return -EINVAL;
> -			}
> -		}
> -
> -		tag = qp_data->temp_digest;
> -		IMB_CHACHA20_POLY1305_DEC_FINALIZE(qp->mb_mgr,
> -					&qp_data-
> >chacha20_poly1305_ctx_data,
> -					tag, session->gen_digest_length);
> -	}
> -
> -	return 0;
> -}
> -
> -/**
> - * Process a completed chacha poly op
> - *
> - * @param qp		Queue Pair to process
> - * @param op		Crypto operation
> - * @param sess		Crypto session
> - *
> - * @return
> - * - void
> - */
> -static void
> -post_process_chacha20_poly1305_crypto_op(struct ipsec_mb_qp *qp,
> -		struct rte_crypto_op *op,
> -		struct chacha20_poly1305_session *session)
> -{
> -	struct chacha20_poly1305_qp_data *qp_data =
> -					ipsec_mb_get_qp_private_data(qp);
> -
> -	op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
> -	/* Verify digest if required */
> -	if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT ||
> -			session->op == IPSEC_MB_OP_HASH_VERIFY_ONLY) {
> -		uint8_t *digest = op->sym->aead.digest.data;
> -		uint8_t *tag = qp_data->temp_digest;
> -
> -#ifdef RTE_LIBRTE_PMD_CHACHA20_POLY1305_DEBUG
> -		rte_hexdump(stdout, "auth tag (orig):",
> -				digest, session->req_digest_length);
> -		rte_hexdump(stdout, "auth tag (calc):",
> -				tag, session->req_digest_length);
> -#endif
> -		if (memcmp(tag, digest,	session->req_digest_length)
> != 0)
> -			op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
> -
> -	}
> -
> -}
> -
> -/**
> - * Process a completed Chacha20_poly1305 request
> - *
> - * @param qp		Queue Pair to process
> - * @param op		Crypto operation
> - * @param sess		Crypto session
> - *
> - * @return
> - * - void
> - */
> -static void
> -handle_completed_chacha20_poly1305_crypto_op(struct ipsec_mb_qp *qp,
> -		struct rte_crypto_op *op,
> -		struct chacha20_poly1305_session *sess)
> -{
> -	post_process_chacha20_poly1305_crypto_op(qp, op, sess);
> -
> -	/* Free session if a session-less crypto op */
> -	if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
> -		memset(sess, 0, sizeof(struct chacha20_poly1305_session));
> -		rte_mempool_put(qp->sess_mp, op->sym->session);
> -		op->sym->session = NULL;
> -	}
> -}
> -
> -static uint16_t
> -chacha20_poly1305_pmd_dequeue_burst(void *queue_pair,
> -		struct rte_crypto_op **ops, uint16_t nb_ops)
> -{
> -	struct chacha20_poly1305_session *sess;
> -	struct ipsec_mb_qp *qp = queue_pair;
> -
> -	int retval = 0;
> -	unsigned int i = 0, nb_dequeued;
> -
> -	nb_dequeued = rte_ring_dequeue_burst(qp->ingress_queue,
> -			(void **)ops, nb_ops, NULL);
> -
> -	for (i = 0; i < nb_dequeued; i++) {
> -
> -		sess = ipsec_mb_get_session_private(qp, ops[i]);
> -		if (unlikely(sess == NULL)) {
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			qp->stats.dequeue_err_count++;
> -			break;
> -		}
> -
> -		retval = chacha20_poly1305_crypto_op(qp, ops[i], sess);
> -		if (retval < 0) {
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			qp->stats.dequeue_err_count++;
> -			break;
> -		}
> -
> -		handle_completed_chacha20_poly1305_crypto_op(qp,
> ops[i], sess);
> -	}
> -
> -	qp->stats.dequeued_count += i;
> -
> -	return i;
> -}
> +#include "pmd_aesni_mb_priv.h"
> 
>  struct rte_cryptodev_ops chacha20_poly1305_pmd_ops = {
>  	.dev_configure = ipsec_mb_config,
> @@ -384,7 +57,7 @@ RTE_INIT(ipsec_mb_register_chacha20_poly1305)
>  		=
> &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_CHACHA20_POLY1305];
> 
>  	chacha_poly_data->caps = chacha20_poly1305_capabilities;
> -	chacha_poly_data->dequeue_burst =
> chacha20_poly1305_pmd_dequeue_burst;
> +	chacha_poly_data->dequeue_burst = aesni_mb_dequeue_burst;
>  	chacha_poly_data->feature_flags =
>  		RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
>  		RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
> @@ -395,10 +68,9 @@ RTE_INIT(ipsec_mb_register_chacha20_poly1305)
>  		RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
>  	chacha_poly_data->internals_priv_size = 0;
>  	chacha_poly_data->ops = &chacha20_poly1305_pmd_ops;
> -	chacha_poly_data->qp_priv_size =
> -			sizeof(struct chacha20_poly1305_qp_data);
> +	chacha_poly_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);
>  	chacha_poly_data->session_configure =
> -			chacha20_poly1305_session_configure;
> +			aesni_mb_session_configure;
>  	chacha_poly_data->session_priv_size =
> -			sizeof(struct chacha20_poly1305_session);
> +			sizeof(struct aesni_mb_session);
>  }
> diff --git a/drivers/crypto/ipsec_mb/pmd_chacha_poly_priv.h
> b/drivers/crypto/ipsec_mb/pmd_chacha_poly_priv.h
> index 842f62f5d1..e668bfe07f 100644
> --- a/drivers/crypto/ipsec_mb/pmd_chacha_poly_priv.h
> +++ b/drivers/crypto/ipsec_mb/pmd_chacha_poly_priv.h
> @@ -7,9 +7,7 @@
> 
>  #include "ipsec_mb_private.h"
> 
> -#define CHACHA20_POLY1305_IV_LENGTH 12
>  #define CHACHA20_POLY1305_DIGEST_LENGTH 16
> -#define CHACHA20_POLY1305_KEY_SIZE  32
> 
>  static const
>  struct rte_cryptodev_capabilities chacha20_poly1305_capabilities[] = {
> @@ -45,30 +43,4 @@ struct rte_cryptodev_capabilities
> chacha20_poly1305_capabilities[] = {
> 
>  uint8_t pmd_driver_id_chacha20_poly1305;
> 
> -/** CHACHA20 POLY1305 private session structure */
> -struct chacha20_poly1305_session {
> -	struct {
> -		uint16_t length;
> -		uint16_t offset;
> -	} iv;
> -	/**< IV parameters */
> -	uint16_t aad_length;
> -	/**< AAD length */
> -	uint16_t req_digest_length;
> -	/**< Requested digest length */
> -	uint16_t gen_digest_length;
> -	/**< Generated digest length */
> -	uint8_t key[CHACHA20_POLY1305_KEY_SIZE];
> -	enum ipsec_mb_operation op;
> -} __rte_cache_aligned;
> -
> -struct chacha20_poly1305_qp_data {
> -	struct chacha20_poly1305_context_data
> chacha20_poly1305_ctx_data;
> -	uint8_t temp_digest[CHACHA20_POLY1305_DIGEST_LENGTH];
> -	/**< Buffer used to store the digest generated
> -	 * by the driver when verifying a digest provided
> -	 * by the user (using authentication verify operation)
> -	 */
> -};
> -
>  #endif /* _PMD_CHACHA_POLY_PRIV_H_ */
> diff --git a/drivers/crypto/ipsec_mb/pmd_kasumi.c
> b/drivers/crypto/ipsec_mb/pmd_kasumi.c
> index 70536ec3dc..c3571ec81b 100644
> --- a/drivers/crypto/ipsec_mb/pmd_kasumi.c
> +++ b/drivers/crypto/ipsec_mb/pmd_kasumi.c
> @@ -10,406 +10,7 @@
>  #include <rte_malloc.h>
> 
>  #include "pmd_kasumi_priv.h"
> -
> -/** Parse crypto xform chain and set private session parameters. */
> -static int
> -kasumi_session_configure(IMB_MGR *mgr, void *priv_sess,
> -			  const struct rte_crypto_sym_xform *xform)
> -{
> -	const struct rte_crypto_sym_xform *auth_xform = NULL;
> -	const struct rte_crypto_sym_xform *cipher_xform = NULL;
> -	enum ipsec_mb_operation mode;
> -	struct kasumi_session *sess = (struct kasumi_session *)priv_sess;
> -	/* Select Crypto operation - hash then cipher / cipher then hash */
> -	int ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
> -				&cipher_xform, NULL);
> -
> -	if (ret)
> -		return ret;
> -
> -	if (cipher_xform) {
> -		/* Only KASUMI F8 supported */
> -		if (cipher_xform->cipher.algo !=
> RTE_CRYPTO_CIPHER_KASUMI_F8) {
> -			IPSEC_MB_LOG(ERR, "Unsupported cipher algorithm
> ");
> -			return -ENOTSUP;
> -		}
> -
> -		sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
> -		if (cipher_xform->cipher.iv.length != KASUMI_IV_LENGTH) {
> -			IPSEC_MB_LOG(ERR, "Wrong IV length");
> -			return -EINVAL;
> -		}
> -
> -		/* Initialize key */
> -		IMB_KASUMI_INIT_F8_KEY_SCHED(mgr,
> -					      cipher_xform->cipher.key.data,
> -					      &sess->pKeySched_cipher);
> -	}
> -
> -	if (auth_xform) {
> -		/* Only KASUMI F9 supported */
> -		if (auth_xform->auth.algo !=
> RTE_CRYPTO_AUTH_KASUMI_F9) {
> -			IPSEC_MB_LOG(ERR, "Unsupported authentication");
> -			return -ENOTSUP;
> -		}
> -
> -		if (auth_xform->auth.digest_length !=
> KASUMI_DIGEST_LENGTH) {
> -			IPSEC_MB_LOG(ERR, "Wrong digest length");
> -			return -EINVAL;
> -		}
> -
> -		sess->auth_op = auth_xform->auth.op;
> -
> -		/* Initialize key */
> -		IMB_KASUMI_INIT_F9_KEY_SCHED(mgr, auth_xform-
> >auth.key.data,
> -					      &sess->pKeySched_hash);
> -	}
> -
> -	sess->op = mode;
> -	return ret;
> -}
> -
> -/** Encrypt/decrypt mbufs with same cipher key. */
> -static uint8_t
> -process_kasumi_cipher_op(struct ipsec_mb_qp *qp, struct rte_crypto_op
> **ops,
> -			  struct kasumi_session *session, uint8_t num_ops)
> -{
> -	unsigned int i;
> -	uint8_t processed_ops = 0;
> -	const void *src[num_ops];
> -	void *dst[num_ops];
> -	uint8_t *iv_ptr;
> -	uint64_t iv[num_ops];
> -	uint32_t num_bytes[num_ops];
> -
> -	for (i = 0; i < num_ops; i++) {
> -		src[i] = rte_pktmbuf_mtod_offset(ops[i]->sym->m_src,
> -						 uint8_t *,
> -						 (ops[i]->sym-
> >cipher.data.offset >> 3));
> -		dst[i] = ops[i]->sym->m_dst
> -			     ? rte_pktmbuf_mtod_offset(ops[i]->sym->m_dst,
> -						       uint8_t *,
> -						       (ops[i]->sym-
> >cipher.data.offset >> 3))
> -			     : rte_pktmbuf_mtod_offset(ops[i]->sym->m_src,
> -						       uint8_t *,
> -						       (ops[i]->sym-
> >cipher.data.offset >> 3));
> -		iv_ptr = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
> -						    session->cipher_iv_offset);
> -		iv[i] = *((uint64_t *)(iv_ptr));
> -		num_bytes[i] = ops[i]->sym->cipher.data.length >> 3;
> -
> -		processed_ops++;
> -	}
> -
> -	if (processed_ops != 0)
> -		IMB_KASUMI_F8_N_BUFFER(qp->mb_mgr, &session-
> >pKeySched_cipher,
> -					iv, src, dst, num_bytes,
> -					processed_ops);
> -
> -	return processed_ops;
> -}
> -
> -/** Encrypt/decrypt mbuf (bit level function). */
> -static uint8_t
> -process_kasumi_cipher_op_bit(struct ipsec_mb_qp *qp, struct
> rte_crypto_op *op,
> -			      struct kasumi_session *session)
> -{
> -	uint8_t *src, *dst;
> -	uint8_t *iv_ptr;
> -	uint64_t iv;
> -	uint32_t length_in_bits, offset_in_bits;
> -
> -	offset_in_bits = op->sym->cipher.data.offset;
> -	src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *);
> -	if (op->sym->m_dst == NULL)
> -		dst = src;
> -	else
> -		dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *);
> -	iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
> -					    session->cipher_iv_offset);
> -	iv = *((uint64_t *)(iv_ptr));
> -	length_in_bits = op->sym->cipher.data.length;
> -
> -	IMB_KASUMI_F8_1_BUFFER_BIT(qp->mb_mgr, &session-
> >pKeySched_cipher, iv,
> -				    src, dst, length_in_bits, offset_in_bits);
> -
> -	return 1;
> -}
> -
> -/** Generate/verify hash from mbufs with same hash key. */
> -static int
> -process_kasumi_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op
> **ops,
> -			struct kasumi_session *session, uint8_t num_ops)
> -{
> -	unsigned int i;
> -	uint8_t processed_ops = 0;
> -	uint8_t *src, *dst;
> -	uint32_t length_in_bits;
> -	uint32_t num_bytes;
> -	struct kasumi_qp_data *qp_data =
> ipsec_mb_get_qp_private_data(qp);
> -
> -	for (i = 0; i < num_ops; i++) {
> -		/* Data must be byte aligned */
> -		if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			IPSEC_MB_LOG(ERR, "Invalid Offset");
> -			break;
> -		}
> -
> -		length_in_bits = ops[i]->sym->auth.data.length;
> -
> -		src = rte_pktmbuf_mtod_offset(ops[i]->sym->m_src, uint8_t
> *,
> -					      (ops[i]->sym->auth.data.offset >>
> 3));
> -		/* Direction from next bit after end of message */
> -		num_bytes = length_in_bits >> 3;
> -
> -		if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) {
> -			dst = qp_data->temp_digest;
> -			IMB_KASUMI_F9_1_BUFFER(qp->mb_mgr,
> -						&session->pKeySched_hash,
> src,
> -						num_bytes, dst);
> -
> -			/* Verify digest. */
> -			if (memcmp(dst, ops[i]->sym->auth.digest.data,
> -				    KASUMI_DIGEST_LENGTH)
> -			    != 0)
> -				ops[i]->status
> -				    = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
> -		} else {
> -			dst = ops[i]->sym->auth.digest.data;
> -
> -			IMB_KASUMI_F9_1_BUFFER(qp->mb_mgr,
> -						&session->pKeySched_hash,
> src,
> -						num_bytes, dst);
> -		}
> -		processed_ops++;
> -	}
> -
> -	return processed_ops;
> -}
> -
> -/** Process a batch of crypto ops which shares the same session. */
> -static int
> -process_ops(struct rte_crypto_op **ops, struct kasumi_session *session,
> -		struct ipsec_mb_qp *qp, uint8_t num_ops)
> -{
> -	unsigned int i;
> -	unsigned int processed_ops;
> -
> -	switch (session->op) {
> -	case IPSEC_MB_OP_ENCRYPT_ONLY:
> -	case IPSEC_MB_OP_DECRYPT_ONLY:
> -		processed_ops
> -		    = process_kasumi_cipher_op(qp, ops, session, num_ops);
> -		break;
> -	case IPSEC_MB_OP_HASH_GEN_ONLY:
> -	case IPSEC_MB_OP_HASH_VERIFY_ONLY:
> -		processed_ops
> -		    = process_kasumi_hash_op(qp, ops, session, num_ops);
> -		break;
> -	case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
> -	case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
> -		processed_ops
> -		    = process_kasumi_cipher_op(qp, ops, session, num_ops);
> -		process_kasumi_hash_op(qp, ops, session, processed_ops);
> -		break;
> -	case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
> -	case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
> -		processed_ops
> -		    = process_kasumi_hash_op(qp, ops, session, num_ops);
> -		process_kasumi_cipher_op(qp, ops, session, processed_ops);
> -		break;
> -	default:
> -		/* Operation not supported. */
> -		processed_ops = 0;
> -	}
> -
> -	for (i = 0; i < num_ops; i++) {
> -		/*
> -		 * If there was no error/authentication failure,
> -		 * change status to successful.
> -		 */
> -		if (ops[i]->status ==
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
> -			ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
> -		/* Free session if a session-less crypto op. */
> -		if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
> -			memset(session, 0, sizeof(struct kasumi_session));
> -			rte_mempool_put(qp->sess_mp, ops[i]->sym-
> >session);
> -			ops[i]->sym->session = NULL;
> -		}
> -	}
> -	return processed_ops;
> -}
> -
> -/** Process a crypto op with length/offset in bits. */
> -static int
> -process_op_bit(struct rte_crypto_op *op, struct kasumi_session *session,
> -		struct ipsec_mb_qp *qp)
> -{
> -	unsigned int processed_op;
> -
> -	switch (session->op) {
> -		/* case KASUMI_OP_ONLY_CIPHER: */
> -	case IPSEC_MB_OP_ENCRYPT_ONLY:
> -	case IPSEC_MB_OP_DECRYPT_ONLY:
> -		processed_op = process_kasumi_cipher_op_bit(qp, op,
> session);
> -		break;
> -	/* case KASUMI_OP_ONLY_AUTH: */
> -	case IPSEC_MB_OP_HASH_GEN_ONLY:
> -	case IPSEC_MB_OP_HASH_VERIFY_ONLY:
> -		processed_op = process_kasumi_hash_op(qp, &op, session,
> 1);
> -		break;
> -	/* case KASUMI_OP_CIPHER_AUTH: */
> -	case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
> -		processed_op = process_kasumi_cipher_op_bit(qp, op,
> session);
> -		if (processed_op == 1)
> -			process_kasumi_hash_op(qp, &op, session, 1);
> -		break;
> -	/* case KASUMI_OP_AUTH_CIPHER: */
> -	case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
> -		processed_op = process_kasumi_hash_op(qp, &op, session,
> 1);
> -		if (processed_op == 1)
> -			process_kasumi_cipher_op_bit(qp, op, session);
> -		break;
> -	default:
> -		/* Operation not supported. */
> -		processed_op = 0;
> -	}
> -
> -	/*
> -	 * If there was no error/authentication failure,
> -	 * change status to successful.
> -	 */
> -	if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
> -		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
> -
> -	/* Free session if a session-less crypto op. */
> -	if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
> -		memset(CRYPTODEV_GET_SYM_SESS_PRIV(op->sym-
> >session), 0,
> -			sizeof(struct kasumi_session));
> -		rte_mempool_put(qp->sess_mp, (void *)op->sym->session);
> -		op->sym->session = NULL;
> -	}
> -	return processed_op;
> -}
> -
> -static uint16_t
> -kasumi_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
> -			  uint16_t nb_ops)
> -{
> -	struct rte_crypto_op *c_ops[nb_ops];
> -	struct rte_crypto_op *curr_c_op = NULL;
> -
> -	struct kasumi_session *prev_sess = NULL, *curr_sess = NULL;
> -	struct ipsec_mb_qp *qp = queue_pair;
> -	unsigned int i;
> -	uint8_t burst_size = 0;
> -	uint8_t processed_ops;
> -	unsigned int nb_dequeued;
> -
> -	nb_dequeued = rte_ring_dequeue_burst(qp->ingress_queue,
> -					      (void **)ops, nb_ops, NULL);
> -	for (i = 0; i < nb_dequeued; i++) {
> -		curr_c_op = ops[i];
> -
> -#ifdef RTE_LIBRTE_PMD_KASUMI_DEBUG
> -		if (!rte_pktmbuf_is_contiguous(curr_c_op->sym->m_src)
> -		    || (curr_c_op->sym->m_dst != NULL
> -			&& !rte_pktmbuf_is_contiguous(
> -			    curr_c_op->sym->m_dst))) {
> -			IPSEC_MB_LOG(ERR,
> -				      "PMD supports only contiguous mbufs, op
> (%p) provides noncontiguous mbuf as source/destination buffer.",
> -				      curr_c_op);
> -			curr_c_op->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			break;
> -		}
> -#endif
> -
> -		/* Set status as enqueued (not processed yet) by default. */
> -		curr_c_op->status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> -
> -		curr_sess = (struct kasumi_session *)
> -			ipsec_mb_get_session_private(qp, curr_c_op);
> -		if (unlikely(curr_sess == NULL
> -			      || curr_sess->op ==
> IPSEC_MB_OP_NOT_SUPPORTED)) {
> -			curr_c_op->status
> -			    = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
> -			break;
> -		}
> -
> -		/* If length/offset is at bit-level, process this buffer alone.
> -		 */
> -		if (((curr_c_op->sym->cipher.data.length % BYTE_LEN) != 0)
> -		    || ((ops[i]->sym->cipher.data.offset % BYTE_LEN) != 0)) {
> -			/* Process the ops of the previous session. */
> -			if (prev_sess != NULL) {
> -				processed_ops = process_ops(c_ops,
> prev_sess,
> -						qp, burst_size);
> -				if (processed_ops < burst_size) {
> -					burst_size = 0;
> -					break;
> -				}
> -
> -				burst_size = 0;
> -				prev_sess = NULL;
> -			}
> -
> -			processed_ops = process_op_bit(curr_c_op,
> -					curr_sess, qp);
> -			if (processed_ops != 1)
> -				break;
> -
> -			continue;
> -		}
> -
> -		/* Batch ops that share the same session. */
> -		if (prev_sess == NULL) {
> -			prev_sess = curr_sess;
> -			c_ops[burst_size++] = curr_c_op;
> -		} else if (curr_sess == prev_sess) {
> -			c_ops[burst_size++] = curr_c_op;
> -			/*
> -			 * When there are enough ops to process in a batch,
> -			 * process them, and start a new batch.
> -			 */
> -			if (burst_size == KASUMI_MAX_BURST) {
> -				processed_ops = process_ops(c_ops,
> prev_sess,
> -						qp, burst_size);
> -				if (processed_ops < burst_size) {
> -					burst_size = 0;
> -					break;
> -				}
> -
> -				burst_size = 0;
> -				prev_sess = NULL;
> -			}
> -		} else {
> -			/*
> -			 * Different session, process the ops
> -			 * of the previous session.
> -			 */
> -			processed_ops = process_ops(c_ops, prev_sess, qp,
> -					burst_size);
> -			if (processed_ops < burst_size) {
> -				burst_size = 0;
> -				break;
> -			}
> -
> -			burst_size = 0;
> -			prev_sess = curr_sess;
> -
> -			c_ops[burst_size++] = curr_c_op;
> -		}
> -	}
> -
> -	if (burst_size != 0) {
> -		/* Process the crypto ops of the last session. */
> -		processed_ops = process_ops(c_ops, prev_sess, qp,
> burst_size);
> -	}
> -
> -	qp->stats.dequeued_count += i;
> -	return i;
> -}
> +#include "pmd_aesni_mb_priv.h"
> 
>  struct rte_cryptodev_ops kasumi_pmd_ops = {
>  	.dev_configure = ipsec_mb_config,
> @@ -460,7 +61,7 @@ RTE_INIT(ipsec_mb_register_kasumi)
>  	    = &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_KASUMI];
> 
>  	kasumi_data->caps = kasumi_capabilities;
> -	kasumi_data->dequeue_burst = kasumi_pmd_dequeue_burst;
> +	kasumi_data->dequeue_burst = aesni_mb_dequeue_burst;
>  	kasumi_data->feature_flags =
> RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO
>  				|
> RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING
>  				|
> RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA
> @@ -469,7 +70,8 @@ RTE_INIT(ipsec_mb_register_kasumi)
>  				| RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
>  	kasumi_data->internals_priv_size = 0;
>  	kasumi_data->ops = &kasumi_pmd_ops;
> -	kasumi_data->qp_priv_size = sizeof(struct kasumi_qp_data);
> -	kasumi_data->session_configure = kasumi_session_configure;
> -	kasumi_data->session_priv_size = sizeof(struct kasumi_session);
> +	kasumi_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);
> +	kasumi_data->session_configure = aesni_mb_session_configure;
> +	kasumi_data->session_priv_size =
> +			sizeof(struct aesni_mb_session);
>  }
> diff --git a/drivers/crypto/ipsec_mb/pmd_kasumi_priv.h
> b/drivers/crypto/ipsec_mb/pmd_kasumi_priv.h
> index 8db1d1cc5b..3223cf1a14 100644
> --- a/drivers/crypto/ipsec_mb/pmd_kasumi_priv.h
> +++ b/drivers/crypto/ipsec_mb/pmd_kasumi_priv.h
> @@ -9,8 +9,6 @@
> 
>  #define KASUMI_KEY_LENGTH 16
>  #define KASUMI_IV_LENGTH 8
> -#define KASUMI_MAX_BURST 4
> -#define BYTE_LEN 8
>  #define KASUMI_DIGEST_LENGTH 4
> 
>  uint8_t pmd_driver_id_kasumi;
> @@ -60,22 +58,4 @@ static const struct rte_cryptodev_capabilities
> kasumi_capabilities[] = {
>  	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
>  };
> 
> -/** KASUMI private session structure */
> -struct kasumi_session {
> -	/* Keys have to be 16-byte aligned */
> -	kasumi_key_sched_t pKeySched_cipher;
> -	kasumi_key_sched_t pKeySched_hash;
> -	enum ipsec_mb_operation op;
> -	enum rte_crypto_auth_operation auth_op;
> -	uint16_t cipher_iv_offset;
> -} __rte_cache_aligned;
> -
> -struct kasumi_qp_data {
> -	uint8_t temp_digest[KASUMI_DIGEST_LENGTH];
> -	/* *< Buffers used to store the digest generated
> -	 * by the driver when verifying a digest provided
> -	 * by the user (using authentication verify operation)
> -	 */
> -};
> -
>  #endif /* _PMD_KASUMI_PRIV_H_ */
> diff --git a/drivers/crypto/ipsec_mb/pmd_snow3g.c
> b/drivers/crypto/ipsec_mb/pmd_snow3g.c
> index a96779f059..957f6aade8 100644
> --- a/drivers/crypto/ipsec_mb/pmd_snow3g.c
> +++ b/drivers/crypto/ipsec_mb/pmd_snow3g.c
> @@ -3,539 +3,7 @@
>   */
> 
>  #include "pmd_snow3g_priv.h"
> -
> -/** Parse crypto xform chain and set private session parameters. */
> -static int
> -snow3g_session_configure(IMB_MGR *mgr, void *priv_sess,
> -		const struct rte_crypto_sym_xform *xform)
> -{
> -	struct snow3g_session *sess = (struct snow3g_session *)priv_sess;
> -	const struct rte_crypto_sym_xform *auth_xform = NULL;
> -	const struct rte_crypto_sym_xform *cipher_xform = NULL;
> -	enum ipsec_mb_operation mode;
> -
> -	/* Select Crypto operation - hash then cipher / cipher then hash */
> -	int ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
> -				&cipher_xform, NULL);
> -	if (ret)
> -		return ret;
> -
> -	if (cipher_xform) {
> -		/* Only SNOW 3G UEA2 supported */
> -		if (cipher_xform->cipher.algo !=
> RTE_CRYPTO_CIPHER_SNOW3G_UEA2)
> -			return -ENOTSUP;
> -
> -		if (cipher_xform->cipher.iv.length != SNOW3G_IV_LENGTH) {
> -			IPSEC_MB_LOG(ERR, "Wrong IV length");
> -			return -EINVAL;
> -		}
> -		if (cipher_xform->cipher.key.length >
> SNOW3G_MAX_KEY_SIZE) {
> -			IPSEC_MB_LOG(ERR, "Not enough memory to store
> the key");
> -			return -ENOMEM;
> -		}
> -
> -		sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
> -
> -		/* Initialize key */
> -		IMB_SNOW3G_INIT_KEY_SCHED(mgr, cipher_xform-
> >cipher.key.data,
> -					&sess->pKeySched_cipher);
> -	}
> -
> -	if (auth_xform) {
> -		/* Only SNOW 3G UIA2 supported */
> -		if (auth_xform->auth.algo !=
> RTE_CRYPTO_AUTH_SNOW3G_UIA2)
> -			return -ENOTSUP;
> -
> -		if (auth_xform->auth.digest_length !=
> SNOW3G_DIGEST_LENGTH) {
> -			IPSEC_MB_LOG(ERR, "Wrong digest length");
> -			return -EINVAL;
> -		}
> -		if (auth_xform->auth.key.length > SNOW3G_MAX_KEY_SIZE) {
> -			IPSEC_MB_LOG(ERR, "Not enough memory to store
> the key");
> -			return -ENOMEM;
> -		}
> -
> -		sess->auth_op = auth_xform->auth.op;
> -
> -		if (auth_xform->auth.iv.length != SNOW3G_IV_LENGTH) {
> -			IPSEC_MB_LOG(ERR, "Wrong IV length");
> -			return -EINVAL;
> -		}
> -		sess->auth_iv_offset = auth_xform->auth.iv.offset;
> -
> -		/* Initialize key */
> -		IMB_SNOW3G_INIT_KEY_SCHED(mgr, auth_xform-
> >auth.key.data,
> -					&sess->pKeySched_hash);
> -	}
> -
> -	sess->op = mode;
> -
> -	return 0;
> -}
> -
> -/** Check if conditions are met for digest-appended operations */
> -static uint8_t *
> -snow3g_digest_appended_in_src(struct rte_crypto_op *op)
> -{
> -	unsigned int auth_size, cipher_size;
> -
> -	auth_size = (op->sym->auth.data.offset >> 3) +
> -		(op->sym->auth.data.length >> 3);
> -	cipher_size = (op->sym->cipher.data.offset >> 3) +
> -		(op->sym->cipher.data.length >> 3);
> -
> -	if (auth_size < cipher_size)
> -		return rte_pktmbuf_mtod_offset(op->sym->m_src,
> -				uint8_t *, auth_size);
> -
> -	return NULL;
> -}
> -
> -/** Encrypt/decrypt mbufs with same cipher key. */
> -static uint8_t
> -process_snow3g_cipher_op(struct ipsec_mb_qp *qp, struct rte_crypto_op
> **ops,
> -		struct snow3g_session *session,
> -		uint8_t num_ops)
> -{
> -	uint32_t i;
> -	uint8_t processed_ops = 0;
> -	const void *src[SNOW3G_MAX_BURST] = {NULL};
> -	void *dst[SNOW3G_MAX_BURST] = {NULL};
> -	uint8_t *digest_appended[SNOW3G_MAX_BURST] = {NULL};
> -	const void *iv[SNOW3G_MAX_BURST] = {NULL};
> -	uint32_t num_bytes[SNOW3G_MAX_BURST] = {0};
> -	uint32_t cipher_off, cipher_len;
> -	int unencrypted_bytes = 0;
> -
> -	for (i = 0; i < num_ops; i++) {
> -
> -		cipher_off = ops[i]->sym->cipher.data.offset >> 3;
> -		cipher_len = ops[i]->sym->cipher.data.length >> 3;
> -		src[i] = rte_pktmbuf_mtod_offset(
> -			ops[i]->sym->m_src,	uint8_t *, cipher_off);
> -
> -		/* If out-of-place operation */
> -		if (ops[i]->sym->m_dst &&
> -			ops[i]->sym->m_src != ops[i]->sym->m_dst) {
> -			dst[i] = rte_pktmbuf_mtod_offset(
> -				ops[i]->sym->m_dst, uint8_t *, cipher_off);
> -
> -			/* In case of out-of-place, auth-cipher operation
> -			 * with partial encryption of the digest, copy
> -			 * the remaining, unencrypted part.
> -			 */
> -			if (session->op ==
> IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT
> -			    || session->op ==
> IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT)
> -				unencrypted_bytes =
> -					(ops[i]->sym->auth.data.offset >> 3) +
> -					(ops[i]->sym->auth.data.length >> 3)
> +
> -					(SNOW3G_DIGEST_LENGTH) -
> -					cipher_off - cipher_len;
> -			if (unencrypted_bytes > 0)
> -				rte_memcpy(
> -					rte_pktmbuf_mtod_offset(
> -						ops[i]->sym->m_dst, uint8_t
> *,
> -						cipher_off + cipher_len),
> -					rte_pktmbuf_mtod_offset(
> -						ops[i]->sym->m_src, uint8_t
> *,
> -						cipher_off + cipher_len),
> -					unencrypted_bytes);
> -		} else
> -			dst[i] = rte_pktmbuf_mtod_offset(ops[i]->sym-
> >m_src,
> -						uint8_t *, cipher_off);
> -
> -		iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
> -				session->cipher_iv_offset);
> -		num_bytes[i] = cipher_len;
> -		processed_ops++;
> -	}
> -
> -	IMB_SNOW3G_F8_N_BUFFER(qp->mb_mgr, &session-
> >pKeySched_cipher, iv,
> -			src, dst, num_bytes, processed_ops);
> -
> -	/* Take care of the raw digest data in src buffer */
> -	for (i = 0; i < num_ops; i++) {
> -		if ((session->op ==
> IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT ||
> -			session->op ==
> IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT) &&
> -				ops[i]->sym->m_dst != NULL) {
> -			digest_appended[i] =
> -				snow3g_digest_appended_in_src(ops[i]);
> -			/* Clear unencrypted digest from
> -			 * the src buffer
> -			 */
> -			if (digest_appended[i] != NULL)
> -				memset(digest_appended[i],
> -					0, SNOW3G_DIGEST_LENGTH);
> -		}
> -	}
> -	return processed_ops;
> -}
> -
> -/** Encrypt/decrypt mbuf (bit level function). */
> -static uint8_t
> -process_snow3g_cipher_op_bit(struct ipsec_mb_qp *qp,
> -		struct rte_crypto_op *op,
> -		struct snow3g_session *session)
> -{
> -	uint8_t *src, *dst;
> -	uint8_t *iv;
> -	uint32_t length_in_bits, offset_in_bits;
> -	int unencrypted_bytes = 0;
> -
> -	offset_in_bits = op->sym->cipher.data.offset;
> -	src = rte_pktmbuf_mtod(op->sym->m_src, uint8_t *);
> -	if (op->sym->m_dst == NULL) {
> -		op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -		IPSEC_MB_LOG(ERR, "bit-level in-place not supported\n");
> -		return 0;
> -	}
> -	length_in_bits = op->sym->cipher.data.length;
> -	dst = rte_pktmbuf_mtod(op->sym->m_dst, uint8_t *);
> -	/* In case of out-of-place, auth-cipher operation
> -	 * with partial encryption of the digest, copy
> -	 * the remaining, unencrypted part.
> -	 */
> -	if (session->op == IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT ||
> -		session->op == IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT)
> -		unencrypted_bytes =
> -			(op->sym->auth.data.offset >> 3) +
> -			(op->sym->auth.data.length >> 3) +
> -			(SNOW3G_DIGEST_LENGTH) -
> -			(offset_in_bits >> 3) -
> -			(length_in_bits >> 3);
> -	if (unencrypted_bytes > 0)
> -		rte_memcpy(
> -			rte_pktmbuf_mtod_offset(
> -				op->sym->m_dst, uint8_t *,
> -				(length_in_bits >> 3)),
> -			rte_pktmbuf_mtod_offset(
> -				op->sym->m_src, uint8_t *,
> -				(length_in_bits >> 3)),
> -				unencrypted_bytes);
> -
> -	iv = rte_crypto_op_ctod_offset(op, uint8_t *,
> -				session->cipher_iv_offset);
> -
> -	IMB_SNOW3G_F8_1_BUFFER_BIT(qp->mb_mgr, &session-
> >pKeySched_cipher, iv,
> -			src, dst, length_in_bits, offset_in_bits);
> -
> -	return 1;
> -}
> -
> -/** Generate/verify hash from mbufs with same hash key. */
> -static int
> -process_snow3g_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op
> **ops,
> -		struct snow3g_session *session,
> -		uint8_t num_ops)
> -{
> -	uint32_t i;
> -	uint8_t processed_ops = 0;
> -	uint8_t *src, *dst;
> -	uint32_t length_in_bits;
> -	uint8_t *iv;
> -	uint8_t digest_appended = 0;
> -	struct snow3g_qp_data *qp_data =
> ipsec_mb_get_qp_private_data(qp);
> -
> -	for (i = 0; i < num_ops; i++) {
> -		/* Data must be byte aligned */
> -		if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			IPSEC_MB_LOG(ERR, "Offset");
> -			break;
> -		}
> -
> -		dst = NULL;
> -
> -		length_in_bits = ops[i]->sym->auth.data.length;
> -
> -		src = rte_pktmbuf_mtod_offset(ops[i]->sym->m_src, uint8_t
> *,
> -					      (ops[i]->sym->auth.data.offset >>
> 3));
> -		iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
> -				session->auth_iv_offset);
> -
> -		if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) {
> -			dst = qp_data->temp_digest;
> -			 /* Handle auth cipher verify oop case*/
> -			if ((session->op ==
> -				IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN
> ||
> -				session->op ==
> -
> 	IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY) &&
> -				ops[i]->sym->m_dst != NULL)
> -				src = rte_pktmbuf_mtod_offset(
> -					ops[i]->sym->m_dst, uint8_t *,
> -					ops[i]->sym->auth.data.offset >> 3);
> -
> -			IMB_SNOW3G_F9_1_BUFFER(qp->mb_mgr,
> -					&session->pKeySched_hash,
> -					iv, src, length_in_bits, dst);
> -			/* Verify digest. */
> -			if (memcmp(dst, ops[i]->sym->auth.digest.data,
> -					SNOW3G_DIGEST_LENGTH) != 0)
> -				ops[i]->status =
> -
> 	RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
> -		} else {
> -			if (session->op ==
> -				IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT
> ||
> -				session->op ==
> -				IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT)
> -				dst =
> snow3g_digest_appended_in_src(ops[i]);
> -
> -			if (dst != NULL)
> -				digest_appended = 1;
> -			else
> -				dst = ops[i]->sym->auth.digest.data;
> -
> -			IMB_SNOW3G_F9_1_BUFFER(qp->mb_mgr,
> -					&session->pKeySched_hash,
> -					iv, src, length_in_bits, dst);
> -
> -			/* Copy back digest from src to auth.digest.data */
> -			if (digest_appended)
> -				rte_memcpy(ops[i]->sym->auth.digest.data,
> -					dst, SNOW3G_DIGEST_LENGTH);
> -		}
> -		processed_ops++;
> -	}
> -
> -	return processed_ops;
> -}
> -
> -/** Process a batch of crypto ops which shares the same session. */
> -static int
> -process_ops(struct rte_crypto_op **ops, struct snow3g_session *session,
> -		struct ipsec_mb_qp *qp, uint8_t num_ops)
> -{
> -	uint32_t i;
> -	uint32_t processed_ops;
> -
> -#ifdef RTE_LIBRTE_PMD_SNOW3G_DEBUG
> -	for (i = 0; i < num_ops; i++) {
> -		if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) ||
> -				(ops[i]->sym->m_dst != NULL &&
> -				!rte_pktmbuf_is_contiguous(
> -						ops[i]->sym->m_dst))) {
> -			IPSEC_MB_LOG(ERR,
> -				"PMD supports only contiguous mbufs, "
> -				"op (%p) provides noncontiguous mbuf as "
> -				"source/destination buffer.\n", ops[i]);
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			return 0;
> -		}
> -	}
> -#endif
> -
> -	switch (session->op) {
> -	case IPSEC_MB_OP_ENCRYPT_ONLY:
> -	case IPSEC_MB_OP_DECRYPT_ONLY:
> -		processed_ops = process_snow3g_cipher_op(qp, ops,
> -				session, num_ops);
> -		break;
> -	case IPSEC_MB_OP_HASH_GEN_ONLY:
> -	case IPSEC_MB_OP_HASH_VERIFY_ONLY:
> -		processed_ops = process_snow3g_hash_op(qp, ops, session,
> -				num_ops);
> -		break;
> -	case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
> -	case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
> -		processed_ops = process_snow3g_cipher_op(qp, ops,
> session,
> -				num_ops);
> -		process_snow3g_hash_op(qp, ops, session, processed_ops);
> -		break;
> -	case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
> -	case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
> -		processed_ops = process_snow3g_hash_op(qp, ops, session,
> -				num_ops);
> -		process_snow3g_cipher_op(qp, ops, session,
> processed_ops);
> -		break;
> -	default:
> -		/* Operation not supported. */
> -		processed_ops = 0;
> -	}
> -
> -	for (i = 0; i < num_ops; i++) {
> -		/*
> -		 * If there was no error/authentication failure,
> -		 * change status to successful.
> -		 */
> -		if (ops[i]->status ==
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
> -			ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
> -		/* Free session if a session-less crypto op. */
> -		if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
> -			memset(session, 0, sizeof(struct snow3g_session));
> -			rte_mempool_put(qp->sess_mp, ops[i]->sym-
> >session);
> -			ops[i]->sym->session = NULL;
> -		}
> -	}
> -	return processed_ops;
> -}
> -
> -/** Process a crypto op with length/offset in bits. */
> -static int
> -process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session,
> -		struct ipsec_mb_qp *qp)
> -{
> -	unsigned int processed_op;
> -	int ret;
> -
> -	switch (session->op) {
> -	case IPSEC_MB_OP_ENCRYPT_ONLY:
> -	case IPSEC_MB_OP_DECRYPT_ONLY:
> -
> -		processed_op = process_snow3g_cipher_op_bit(qp, op,
> -				session);
> -		break;
> -	case IPSEC_MB_OP_HASH_GEN_ONLY:
> -	case IPSEC_MB_OP_HASH_VERIFY_ONLY:
> -		processed_op = process_snow3g_hash_op(qp, &op, session,
> 1);
> -		break;
> -	case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
> -	case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
> -		processed_op = process_snow3g_cipher_op_bit(qp, op,
> session);
> -		if (processed_op == 1)
> -			process_snow3g_hash_op(qp, &op, session, 1);
> -		break;
> -	case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
> -	case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
> -		processed_op = process_snow3g_hash_op(qp, &op, session,
> 1);
> -		if (processed_op == 1)
> -			process_snow3g_cipher_op_bit(qp, op, session);
> -		break;
> -	default:
> -		/* Operation not supported. */
> -		processed_op = 0;
> -	}
> -
> -	/*
> -	 * If there was no error/authentication failure,
> -	 * change status to successful.
> -	 */
> -	if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
> -		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
> -
> -	/* Free session if a session-less crypto op. */
> -	if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
> -		memset(CRYPTODEV_GET_SYM_SESS_PRIV(op->sym-
> >session), 0,
> -			sizeof(struct snow3g_session));
> -		rte_mempool_put(qp->sess_mp, (void *)op->sym->session);
> -		op->sym->session = NULL;
> -	}
> -
> -	if (unlikely(processed_op != 1))
> -		return 0;
> -
> -	ret = rte_ring_enqueue(qp->ingress_queue, op);
> -	if (ret != 0)
> -		return ret;
> -
> -	return 1;
> -}
> -
> -static uint16_t
> -snow3g_pmd_dequeue_burst(void *queue_pair,
> -		struct rte_crypto_op **ops, uint16_t nb_ops)
> -{
> -	struct ipsec_mb_qp *qp = queue_pair;
> -	struct rte_crypto_op *c_ops[SNOW3G_MAX_BURST];
> -	struct rte_crypto_op *curr_c_op;
> -
> -	struct snow3g_session *prev_sess = NULL, *curr_sess = NULL;
> -	uint32_t i;
> -	uint8_t burst_size = 0;
> -	uint8_t processed_ops;
> -	uint32_t nb_dequeued;
> -
> -	nb_dequeued = rte_ring_dequeue_burst(qp->ingress_queue,
> -			(void **)ops, nb_ops, NULL);
> -
> -	for (i = 0; i < nb_dequeued; i++) {
> -		curr_c_op = ops[i];
> -
> -		/* Set status as enqueued (not processed yet) by default. */
> -		curr_c_op->status =
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
> -
> -		curr_sess = ipsec_mb_get_session_private(qp, curr_c_op);
> -		if (unlikely(curr_sess == NULL ||
> -				curr_sess->op ==
> IPSEC_MB_OP_NOT_SUPPORTED)) {
> -			curr_c_op->status =
> -
> 	RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
> -			break;
> -		}
> -
> -		/* If length/offset is at bit-level,
> -		 * process this buffer alone.
> -		 */
> -		if (((curr_c_op->sym->cipher.data.length % BYTE_LEN) != 0)
> -				|| ((curr_c_op->sym->cipher.data.offset
> -					% BYTE_LEN) != 0)) {
> -			/* Process the ops of the previous session. */
> -			if (prev_sess != NULL) {
> -				processed_ops = process_ops(c_ops,
> prev_sess,
> -						qp, burst_size);
> -				if (processed_ops < burst_size) {
> -					burst_size = 0;
> -					break;
> -				}
> -
> -				burst_size = 0;
> -				prev_sess = NULL;
> -			}
> -
> -			processed_ops = process_op_bit(curr_c_op,
> curr_sess, qp);
> -			if (processed_ops != 1)
> -				break;
> -
> -			continue;
> -		}
> -
> -		/* Batch ops that share the same session. */
> -		if (prev_sess == NULL) {
> -			prev_sess = curr_sess;
> -			c_ops[burst_size++] = curr_c_op;
> -		} else if (curr_sess == prev_sess) {
> -			c_ops[burst_size++] = curr_c_op;
> -			/*
> -			 * When there are enough ops to process in a batch,
> -			 * process them, and start a new batch.
> -			 */
> -			if (burst_size == SNOW3G_MAX_BURST) {
> -				processed_ops = process_ops(c_ops,
> prev_sess,
> -						qp, burst_size);
> -				if (processed_ops < burst_size) {
> -					burst_size = 0;
> -					break;
> -				}
> -
> -				burst_size = 0;
> -				prev_sess = NULL;
> -			}
> -		} else {
> -			/*
> -			 * Different session, process the ops
> -			 * of the previous session.
> -			 */
> -			processed_ops = process_ops(c_ops, prev_sess,
> -					qp, burst_size);
> -			if (processed_ops < burst_size) {
> -				burst_size = 0;
> -				break;
> -			}
> -
> -			burst_size = 0;
> -			prev_sess = curr_sess;
> -
> -			c_ops[burst_size++] = curr_c_op;
> -		}
> -	}
> -
> -	if (burst_size != 0) {
> -		/* Process the crypto ops of the last session. */
> -		processed_ops = process_ops(c_ops, prev_sess,
> -				qp, burst_size);
> -	}
> -
> -	qp->stats.dequeued_count += i;
> -	return i;
> -}
> +#include "pmd_aesni_mb_priv.h"
> 
>  struct rte_cryptodev_ops snow3g_pmd_ops = {
>  	.dev_configure = ipsec_mb_config,
> @@ -586,7 +54,7 @@ RTE_INIT(ipsec_mb_register_snow3g)
>  		= &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_SNOW3G];
> 
>  	snow3g_data->caps = snow3g_capabilities;
> -	snow3g_data->dequeue_burst = snow3g_pmd_dequeue_burst;
> +	snow3g_data->dequeue_burst = aesni_mb_dequeue_burst;
>  	snow3g_data->feature_flags =
> RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
>  			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
>  			RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
> @@ -595,7 +63,8 @@ RTE_INIT(ipsec_mb_register_snow3g)
>  			RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED;
>  	snow3g_data->internals_priv_size = 0;
>  	snow3g_data->ops = &snow3g_pmd_ops;
> -	snow3g_data->qp_priv_size = sizeof(struct snow3g_qp_data);
> -	snow3g_data->session_configure = snow3g_session_configure;
> -	snow3g_data->session_priv_size = sizeof(struct snow3g_session);
> +	snow3g_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);
> +	snow3g_data->session_configure = aesni_mb_session_configure;
> +	snow3g_data->session_priv_size =
> +			sizeof(struct aesni_mb_session);
>  }
> diff --git a/drivers/crypto/ipsec_mb/pmd_snow3g_priv.h
> b/drivers/crypto/ipsec_mb/pmd_snow3g_priv.h
> index ca1ce7f9d6..3ceb33b602 100644
> --- a/drivers/crypto/ipsec_mb/pmd_snow3g_priv.h
> +++ b/drivers/crypto/ipsec_mb/pmd_snow3g_priv.h
> @@ -8,10 +8,7 @@
>  #include "ipsec_mb_private.h"
> 
>  #define SNOW3G_IV_LENGTH 16
> -#define SNOW3G_MAX_BURST 8
> -#define BYTE_LEN 8
>  #define SNOW3G_DIGEST_LENGTH 4
> -#define SNOW3G_MAX_KEY_SIZE  128
> 
>  uint8_t pmd_driver_id_snow3g;
> 
> @@ -64,22 +61,4 @@ static const struct rte_cryptodev_capabilities
> snow3g_capabilities[] = {
>  	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
>  };
> 
> -/** SNOW 3G private session structure */
> -struct snow3g_session {
> -	enum ipsec_mb_operation op;
> -	enum rte_crypto_auth_operation auth_op;
> -	snow3g_key_schedule_t pKeySched_cipher;
> -	snow3g_key_schedule_t pKeySched_hash;
> -	uint16_t cipher_iv_offset;
> -	uint16_t auth_iv_offset;
> -} __rte_cache_aligned;
> -
> -struct snow3g_qp_data {
> -	uint8_t temp_digest[SNOW3G_DIGEST_LENGTH];
> -	/**< Buffer used to store the digest generated
> -	 * by the driver when verifying a digest provided
> -	 * by the user (using authentication verify operation)
> -	 */
> -};
> -
>  #endif /* _PMD_SNOW3G_PRIV_H_ */
> diff --git a/drivers/crypto/ipsec_mb/pmd_zuc.c
> b/drivers/crypto/ipsec_mb/pmd_zuc.c
> index 44781be1d1..b72191c7a7 100644
> --- a/drivers/crypto/ipsec_mb/pmd_zuc.c
> +++ b/drivers/crypto/ipsec_mb/pmd_zuc.c
> @@ -3,343 +3,7 @@
>   */
> 
>  #include "pmd_zuc_priv.h"
> -
> -/** Parse crypto xform chain and set private session parameters. */
> -static int
> -zuc_session_configure(__rte_unused IMB_MGR * mgr, void *zuc_sess,
> -		const struct rte_crypto_sym_xform *xform)
> -{
> -	struct zuc_session *sess = (struct zuc_session *) zuc_sess;
> -	const struct rte_crypto_sym_xform *auth_xform = NULL;
> -	const struct rte_crypto_sym_xform *cipher_xform = NULL;
> -	enum ipsec_mb_operation mode;
> -	/* Select Crypto operation - hash then cipher / cipher then hash */
> -	int ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
> -				&cipher_xform, NULL);
> -
> -	if (ret)
> -		return ret;
> -
> -	if (cipher_xform) {
> -		/* Only ZUC EEA3 supported */
> -		if (cipher_xform->cipher.algo !=
> RTE_CRYPTO_CIPHER_ZUC_EEA3)
> -			return -ENOTSUP;
> -
> -		if (cipher_xform->cipher.iv.length != ZUC_IV_KEY_LENGTH) {
> -			IPSEC_MB_LOG(ERR, "Wrong IV length");
> -			return -EINVAL;
> -		}
> -		sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
> -
> -		/* Copy the key */
> -		memcpy(sess->pKey_cipher, cipher_xform->cipher.key.data,
> -				ZUC_IV_KEY_LENGTH);
> -	}
> -
> -	if (auth_xform) {
> -		/* Only ZUC EIA3 supported */
> -		if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_ZUC_EIA3)
> -			return -ENOTSUP;
> -
> -		if (auth_xform->auth.digest_length != ZUC_DIGEST_LENGTH) {
> -			IPSEC_MB_LOG(ERR, "Wrong digest length");
> -			return -EINVAL;
> -		}
> -
> -		sess->auth_op = auth_xform->auth.op;
> -
> -		if (auth_xform->auth.iv.length != ZUC_IV_KEY_LENGTH) {
> -			IPSEC_MB_LOG(ERR, "Wrong IV length");
> -			return -EINVAL;
> -		}
> -		sess->auth_iv_offset = auth_xform->auth.iv.offset;
> -
> -		/* Copy the key */
> -		memcpy(sess->pKey_hash, auth_xform->auth.key.data,
> -				ZUC_IV_KEY_LENGTH);
> -	}
> -
> -	sess->op = mode;
> -	return 0;
> -}
> -
> -/** Encrypt/decrypt mbufs. */
> -static uint8_t
> -process_zuc_cipher_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
> -		struct zuc_session **sessions,
> -		uint8_t num_ops)
> -{
> -	unsigned int i;
> -	uint8_t processed_ops = 0;
> -	const void *src[ZUC_MAX_BURST];
> -	void *dst[ZUC_MAX_BURST];
> -	const void *iv[ZUC_MAX_BURST];
> -	uint32_t num_bytes[ZUC_MAX_BURST];
> -	const void *cipher_keys[ZUC_MAX_BURST];
> -	struct zuc_session *sess;
> -
> -	for (i = 0; i < num_ops; i++) {
> -		if (((ops[i]->sym->cipher.data.length % BYTE_LEN) != 0)
> -				|| ((ops[i]->sym->cipher.data.offset
> -					% BYTE_LEN) != 0)) {
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			IPSEC_MB_LOG(ERR, "Data Length or offset");
> -			break;
> -		}
> -
> -		sess = sessions[i];
> -
> -#ifdef RTE_LIBRTE_PMD_ZUC_DEBUG
> -		if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) ||
> -				(ops[i]->sym->m_dst != NULL &&
> -				!rte_pktmbuf_is_contiguous(
> -						ops[i]->sym->m_dst))) {
> -			IPSEC_MB_LOG(ERR, "PMD supports only "
> -				" contiguous mbufs, op (%p) "
> -				"provides noncontiguous mbuf "
> -				"as source/destination buffer.\n",
> -				"PMD supports only contiguous mbufs, "
> -				"op (%p) provides noncontiguous mbuf "
> -				"as source/destination buffer.\n",
> -				ops[i]);
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			break;
> -		}
> -#endif
> -
> -		src[i] = rte_pktmbuf_mtod_offset(ops[i]->sym->m_src,
> -						 uint8_t *,
> -						 (ops[i]->sym-
> >cipher.data.offset >> 3));
> -		dst[i] = ops[i]->sym->m_dst ?
> -			rte_pktmbuf_mtod_offset(ops[i]->sym->m_dst,
> uint8_t *,
> -						(ops[i]->sym-
> >cipher.data.offset >> 3)) :
> -			rte_pktmbuf_mtod_offset(ops[i]->sym->m_src,
> uint8_t *,
> -						(ops[i]->sym-
> >cipher.data.offset >> 3));
> -		iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
> -				sess->cipher_iv_offset);
> -		num_bytes[i] = ops[i]->sym->cipher.data.length >> 3;
> -
> -		cipher_keys[i] = sess->pKey_cipher;
> -
> -		processed_ops++;
> -	}
> -
> -	IMB_ZUC_EEA3_N_BUFFER(qp->mb_mgr, (const void **)cipher_keys,
> -			(const void **)iv, (const void **)src, (void **)dst,
> -			num_bytes, processed_ops);
> -
> -	return processed_ops;
> -}
> -
> -/** Generate/verify hash from mbufs. */
> -static int
> -process_zuc_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
> -		struct zuc_session **sessions,
> -		uint8_t num_ops)
> -{
> -	unsigned int i;
> -	uint8_t processed_ops = 0;
> -	uint8_t *src[ZUC_MAX_BURST] = { 0 };
> -	uint32_t *dst[ZUC_MAX_BURST];
> -	uint32_t length_in_bits[ZUC_MAX_BURST] = { 0 };
> -	uint8_t *iv[ZUC_MAX_BURST] = { 0 };
> -	const void *hash_keys[ZUC_MAX_BURST] = { 0 };
> -	struct zuc_session *sess;
> -	struct zuc_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
> -
> -
> -	for (i = 0; i < num_ops; i++) {
> -		/* Data must be byte aligned */
> -		if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
> -			IPSEC_MB_LOG(ERR, "Offset");
> -			break;
> -		}
> -
> -		sess = sessions[i];
> -
> -		length_in_bits[i] = ops[i]->sym->auth.data.length;
> -
> -		src[i] = rte_pktmbuf_mtod_offset(ops[i]->sym->m_src,
> -						 uint8_t *,
> -						 (ops[i]->sym-
> >auth.data.offset >> 3));
> -		iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
> -				sess->auth_iv_offset);
> -
> -		hash_keys[i] = sess->pKey_hash;
> -		if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
> -			dst[i] = (uint32_t *)qp_data->temp_digest[i];
> -		else
> -			dst[i] = (uint32_t *)ops[i]->sym->auth.digest.data;
> -
> -		processed_ops++;
> -	}
> -
> -	IMB_ZUC_EIA3_N_BUFFER(qp->mb_mgr, (const void **)hash_keys,
> -			(const void * const *)iv, (const void * const *)src,
> -			length_in_bits, dst, processed_ops);
> -
> -	/*
> -	 * If tag needs to be verified, compare generated tag
> -	 * with attached tag
> -	 */
> -	for (i = 0; i < processed_ops; i++)
> -		if (sessions[i]->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
> -			if (memcmp(dst[i], ops[i]->sym->auth.digest.data,
> -					ZUC_DIGEST_LENGTH) != 0)
> -				ops[i]->status =
> -
> 	RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
> -
> -	return processed_ops;
> -}
> -
> -/** Process a batch of crypto ops which shares the same operation type. */
> -static int
> -process_ops(struct rte_crypto_op **ops, enum ipsec_mb_operation
> op_type,
> -		struct zuc_session **sessions,
> -		struct ipsec_mb_qp *qp, uint8_t num_ops)
> -{
> -	unsigned int i;
> -	unsigned int processed_ops = 0;
> -
> -	switch (op_type) {
> -	case IPSEC_MB_OP_ENCRYPT_ONLY:
> -	case IPSEC_MB_OP_DECRYPT_ONLY:
> -		processed_ops = process_zuc_cipher_op(qp, ops,
> -				sessions, num_ops);
> -		break;
> -	case IPSEC_MB_OP_HASH_GEN_ONLY:
> -	case IPSEC_MB_OP_HASH_VERIFY_ONLY:
> -		processed_ops = process_zuc_hash_op(qp, ops, sessions,
> -				num_ops);
> -		break;
> -	case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
> -	case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
> -		processed_ops = process_zuc_cipher_op(qp, ops, sessions,
> -				num_ops);
> -		process_zuc_hash_op(qp, ops, sessions, processed_ops);
> -		break;
> -	case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
> -	case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
> -		processed_ops = process_zuc_hash_op(qp, ops, sessions,
> -				num_ops);
> -		process_zuc_cipher_op(qp, ops, sessions, processed_ops);
> -		break;
> -	default:
> -		/* Operation not supported. */
> -		for (i = 0; i < num_ops; i++)
> -			ops[i]->status =
> RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
> -	}
> -
> -	for (i = 0; i < num_ops; i++) {
> -		/*
> -		 * If there was no error/authentication failure,
> -		 * change status to successful.
> -		 */
> -		if (ops[i]->status ==
> RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
> -			ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
> -		/* Free session if a session-less crypto op. */
> -		if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
> -			memset(sessions[i], 0, sizeof(struct zuc_session));
> -			rte_mempool_put(qp->sess_mp, ops[i]->sym-
> >session);
> -			ops[i]->sym->session = NULL;
> -		}
> -	}
> -	return processed_ops;
> -}
> -
> -static uint16_t
> -zuc_pmd_dequeue_burst(void *queue_pair,
> -		struct rte_crypto_op **c_ops, uint16_t nb_ops)
> -{
> -
> -	struct rte_crypto_op *curr_c_op;
> -
> -	struct zuc_session *curr_sess;
> -	struct zuc_session *sessions[ZUC_MAX_BURST];
> -	struct rte_crypto_op *int_c_ops[ZUC_MAX_BURST];
> -	enum ipsec_mb_operation prev_zuc_op =
> IPSEC_MB_OP_NOT_SUPPORTED;
> -	enum ipsec_mb_operation curr_zuc_op;
> -	struct ipsec_mb_qp *qp = queue_pair;
> -	unsigned int nb_dequeued;
> -	unsigned int i;
> -	uint8_t burst_size = 0;
> -	uint8_t processed_ops;
> -
> -	nb_dequeued = rte_ring_dequeue_burst(qp->ingress_queue,
> -			(void **)c_ops, nb_ops, NULL);
> -
> -
> -	for (i = 0; i < nb_dequeued; i++) {
> -		curr_c_op = c_ops[i];
> -
> -		curr_sess = (struct zuc_session *)
> -			ipsec_mb_get_session_private(qp, curr_c_op);
> -		if (unlikely(curr_sess == NULL)) {
> -			curr_c_op->status =
> -
> 	RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
> -			break;
> -		}
> -
> -		curr_zuc_op = curr_sess->op;
> -
> -		/*
> -		 * Batch ops that share the same operation type
> -		 * (cipher only, auth only...).
> -		 */
> -		if (burst_size == 0) {
> -			prev_zuc_op = curr_zuc_op;
> -			int_c_ops[0] = curr_c_op;
> -			sessions[0] = curr_sess;
> -			burst_size++;
> -		} else if (curr_zuc_op == prev_zuc_op) {
> -			int_c_ops[burst_size] = curr_c_op;
> -			sessions[burst_size] = curr_sess;
> -			burst_size++;
> -			/*
> -			 * When there are enough ops to process in a batch,
> -			 * process them, and start a new batch.
> -			 */
> -			if (burst_size == ZUC_MAX_BURST) {
> -				processed_ops = process_ops(int_c_ops,
> curr_zuc_op,
> -						sessions, qp, burst_size);
> -				if (processed_ops < burst_size) {
> -					burst_size = 0;
> -					break;
> -				}
> -
> -				burst_size = 0;
> -			}
> -		} else {
> -			/*
> -			 * Different operation type, process the ops
> -			 * of the previous type.
> -			 */
> -			processed_ops = process_ops(int_c_ops,
> prev_zuc_op,
> -					sessions, qp, burst_size);
> -			if (processed_ops < burst_size) {
> -				burst_size = 0;
> -				break;
> -			}
> -
> -			burst_size = 0;
> -			prev_zuc_op = curr_zuc_op;
> -
> -			int_c_ops[0] = curr_c_op;
> -			sessions[0] = curr_sess;
> -			burst_size++;
> -		}
> -	}
> -
> -	if (burst_size != 0) {
> -		/* Process the crypto ops of the last operation type. */
> -		processed_ops = process_ops(int_c_ops, prev_zuc_op,
> -				sessions, qp, burst_size);
> -	}
> -
> -	qp->stats.dequeued_count += i;
> -	return i;
> -}
> +#include "pmd_aesni_mb_priv.h"
> 
>  struct rte_cryptodev_ops zuc_pmd_ops = {
>  	.dev_configure = ipsec_mb_config,
> @@ -390,7 +54,7 @@ RTE_INIT(ipsec_mb_register_zuc)
>  	    = &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_ZUC];
> 
>  	zuc_data->caps = zuc_capabilities;
> -	zuc_data->dequeue_burst = zuc_pmd_dequeue_burst;
> +	zuc_data->dequeue_burst = aesni_mb_dequeue_burst;
>  	zuc_data->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO
>  			| RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING
>  			| RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA
> @@ -399,7 +63,8 @@ RTE_INIT(ipsec_mb_register_zuc)
>  			| RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
>  	zuc_data->internals_priv_size = 0;
>  	zuc_data->ops = &zuc_pmd_ops;
> -	zuc_data->qp_priv_size = sizeof(struct zuc_qp_data);
> -	zuc_data->session_configure = zuc_session_configure;
> -	zuc_data->session_priv_size = sizeof(struct zuc_session);
> +	zuc_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);
> +	zuc_data->session_configure = aesni_mb_session_configure;
> +	zuc_data->session_priv_size =
> +			sizeof(struct aesni_mb_session);
>  }
> diff --git a/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
> b/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
> index 76fd6758c2..a1e8e3aade 100644
> --- a/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
> +++ b/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
> @@ -10,7 +10,6 @@
>  #define ZUC_IV_KEY_LENGTH 16
>  #define ZUC_DIGEST_LENGTH 4
>  #define ZUC_MAX_BURST 16
> -#define BYTE_LEN 8
> 
>  uint8_t pmd_driver_id_zuc;
> 
> @@ -63,23 +62,4 @@ static const struct rte_cryptodev_capabilities
> zuc_capabilities[] = {
>  	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
>  };
> 
> -/** ZUC private session structure */
> -struct zuc_session {
> -	enum ipsec_mb_operation op;
> -	enum rte_crypto_auth_operation auth_op;
> -	uint8_t pKey_cipher[ZUC_IV_KEY_LENGTH];
> -	uint8_t pKey_hash[ZUC_IV_KEY_LENGTH];
> -	uint16_t cipher_iv_offset;
> -	uint16_t auth_iv_offset;
> -} __rte_cache_aligned;
> -
> -struct zuc_qp_data {
> -
> -	uint8_t temp_digest[ZUC_MAX_BURST][ZUC_DIGEST_LENGTH];
> -	/* *< Buffers used to store the digest generated
> -	 * by the driver when verifying a digest provided
> -	 * by the user (using authentication verify operation)
> -	 */
> -};
> -
>  #endif /* _PMD_ZUC_PRIV_H_ */
> --
> 2.25.1