[dpdk-dev] [PATCH] raw/aesni_mb: introduce aesni_mb rawdev pmd

[Mairtin o Loingsigh <mairtin.oloingsigh@intel.com>]

Exposes AESNI-MB via rawdev API to allow the submission of
operations requiring error detect and other non symmetric
crypto transforms.

Signed-off-by: Mairtin o Loingsigh <mairtin.oloingsigh@intel.com>
---
 config/common_base                                 |    5 +
 drivers/raw/Makefile                               |    2 +
 drivers/raw/aesni_mb/Makefile                      |   43 +
 drivers/raw/aesni_mb/aesni_mb_rawdev.c             | 1215 ++++++++++++++++++++
 drivers/raw/aesni_mb/aesni_mb_rawdev_private.h     |  121 ++
 drivers/raw/aesni_mb/meson.build                   |   26 +
 .../aesni_mb/rte_pmd_rawdev_aesni_mb_version.map   |    3 +
 mk/rte.app.mk                                      |    5 +
 8 files changed, 1420 insertions(+), 0 deletions(-)
 create mode 100644 drivers/raw/aesni_mb/Makefile
 create mode 100644 drivers/raw/aesni_mb/aesni_mb_rawdev.c
 create mode 100644 drivers/raw/aesni_mb/aesni_mb_rawdev_private.h
 create mode 100644 drivers/raw/aesni_mb/meson.build
 create mode 100644 drivers/raw/aesni_mb/rte_pmd_rawdev_aesni_mb_version.map

diff --git a/config/common_base b/config/common_base
index 7ca2f28..b2dbbb7 100644
--- a/config/common_base
+++ b/config/common_base
@@ -820,6 +820,11 @@ CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EP_RAWDEV=y
 CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV=y
 
 #
+# Compile PMD for AESNI raw device
+#
+CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV=n
+
+#
 # Compile librte_ring
 #
 CONFIG_RTE_LIBRTE_RING=y
diff --git a/drivers/raw/Makefile b/drivers/raw/Makefile
index 80b043e..c00a12a 100644
--- a/drivers/raw/Makefile
+++ b/drivers/raw/Makefile
@@ -14,5 +14,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += ioat
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV) += octeontx2_dma
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EP_RAWDEV) += octeontx2_ep
+DIRS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV) += aesni_mb
+DEPDIRS-aesni_mb := common
 
 include $(RTE_SDK)/mk/rte.subdir.mk
diff --git a/drivers/raw/aesni_mb/Makefile b/drivers/raw/aesni_mb/Makefile
new file mode 100644
index 0000000..75c6cad
--- /dev/null
+++ b/drivers/raw/aesni_mb/Makefile
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2015-2018 Intel Corporation
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_pmd_aesni_mb_rawdev.a
+
+# build flags
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+# versioning export map
+EXPORT_MAP := rte_pmd_rawdev_aesni_mb_version.map
+
+# external library dependencies
+LDLIBS += -lIPSec_MB
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_cryptodev
+LDLIBS += -lrte_rawdev
+LDLIBS += -lrte_bus_vdev
+LDLIBS += -lrte_multi_fn
+
+IMB_HDR = $(shell echo '\#include <intel-ipsec-mb.h>' | \
+	$(CC) -E $(EXTRA_CFLAGS) - | grep 'intel-ipsec-mb.h' | \
+	head -n1 | cut -d'"' -f2)
+
+# Detect library version
+IMB_VERSION = $(shell grep -e "IMB_VERSION_STR" $(IMB_HDR) | cut -d'"' -f2)
+IMB_VERSION_NUM = $(shell grep -e "IMB_VERSION_NUM" $(IMB_HDR) | cut -d' ' -f3)
+
+ifeq ($(IMB_VERSION),)
+$(error "IPSec_MB version >= 0.53.3 is required")
+endif
+
+ifeq ($(shell expr $(IMB_VERSION_NUM) \< 0x3503), 1)
+$(error "IPSec_MB version >= 0.53.3 is required")
+endif
+
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV) += aesni_mb_rawdev.c
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/raw/aesni_mb/aesni_mb_rawdev.c b/drivers/raw/aesni_mb/aesni_mb_rawdev.c
new file mode 100644
index 0000000..576ed84
--- /dev/null
+++ b/drivers/raw/aesni_mb/aesni_mb_rawdev.c
@@ -0,0 +1,1215 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include <intel-ipsec-mb.h>
+
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_cryptodev.h>
+#include <rte_dev.h>
+#include <rte_eal.h>
+#include <rte_bus_vdev.h>
+#include <rte_malloc.h>
+#include <rte_cpuflags.h>
+#include <rte_rawdev.h>
+#include <rte_rawdev_pmd.h>
+#include <rte_string_fns.h>
+#include <rte_multi_fn.h>
+
+#include "aesni_mb_rawdev_private.h"
+
+#define AESNI_MB_RAWDEV_PMD_SOCKET_ID_ARG ("socket_id")
+
+static const char * const rawdev_pmd_valid_params[] = {
+		AESNI_MB_RAWDEV_PMD_SOCKET_ID_ARG
+};
+
+static const unsigned err_detect_output_byte_lengths[] = {
+		[IMB_AUTH_DOCSIS_CRC32]  = 4,
+};
+
+static inline unsigned
+get_output_byte_length(JOB_HASH_ALG algo)
+{
+	return err_detect_output_byte_lengths[algo];
+}
+
+int
+pmd_parse_uint_arg(const char *key __rte_unused,
+		const char *value, void *extra_args);
+
+int
+parse_input_args(unsigned int *socket_id,
+		const char *args);
+
+static int
+aesni_mb_rawdev_pmd_config(const struct rte_rawdev *dev,
+		rte_rawdev_obj_t config)
+{
+	struct rte_multi_fn_device_info *dev_priv;
+	struct rte_multi_fn_dev_config *conf;
+	struct aesni_mb_rawdev_dev *d;
+
+	dev_priv = config;
+	conf = dev_priv->config;
+	d = dev->dev_private;
+	d->num_queue_pair = conf->nb_queues;
+
+	d->queue_pairs = rte_zmalloc_socket("aesni_rawdev_qps",
+			d->num_queue_pair * sizeof(struct aesni_mb_rawdev_qp),
+			RTE_CACHE_LINE_SIZE, dev->socket_id);
+
+	if (!d->queue_pairs) {
+		AESNI_MB_RAWDEV_LOG(ERR, "Unable to allocate queue pairs");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int
+aesni_mb_rawdev_pmd_start(__rte_unused struct rte_rawdev *dev)
+{
+	return 0;
+}
+
+static void
+aesni_mb_rawdev_pmd_stop(__rte_unused struct rte_rawdev *dev)
+{
+}
+
+static int
+aesni_mb_rawdev_pmd_close(__rte_unused struct rte_rawdev *dev)
+{
+	return 0;
+}
+
+static int
+set_session_err_detect_parameters(struct aesni_mb_rawdev_session *sess,
+		struct rte_multi_fn_err_detect_xform *xform)
+{
+
+	if (xform == NULL) {
+		AESNI_MB_RAWDEV_LOG(ERR, "Invalid error detection xform");
+		return -EINVAL;
+	}
+
+	/* Select error detect generate/verify */
+	if (xform->op == RTE_MULTI_FN_ERR_DETECT_OP_VERIFY)
+		sess->err_detect.operation =
+				RTE_MULTI_FN_ERR_DETECT_OP_VERIFY;
+	else if (xform->op == RTE_MULTI_FN_ERR_DETECT_OP_GENERATE)
+		sess->err_detect.operation =
+				RTE_MULTI_FN_ERR_DETECT_OP_GENERATE;
+	else {
+		AESNI_MB_RAWDEV_LOG(ERR, "Unsupported err_detect operation");
+		return -ENOTSUP;
+	}
+
+	if (xform->algo == RTE_MULTI_FN_ERR_DETECT_CRC32_ETH) {
+		sess->err_detect.algo = IMB_AUTH_DOCSIS_CRC32;
+	} else {
+		AESNI_MB_RAWDEV_LOG(ERR,
+			"Unsupported error detect algorithm selection");
+		return -ENOTSUP;
+	}
+
+	sess->err_detect.gen_output_len =
+			get_output_byte_length(sess->err_detect.algo);
+
+	return 0;
+}
+
+static int
+set_session_cipher_parameters(const MB_MGR *mb_mgr,
+		struct aesni_mb_rawdev_session *sess,
+		const struct rte_crypto_sym_xform *xform)
+{
+
+	if (xform == NULL) {
+		sess->cipher.mode = IMB_CIPHER_NULL;
+		return -EINVAL;
+	}
+
+	if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
+		AESNI_MB_RAWDEV_LOG(ERR, "Crypto xform not of type cipher");
+		return -EINVAL;
+	}
+
+	/* Select cipher direction */
+	switch (xform->cipher.op) {
+	case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
+		sess->cipher.direction = IMB_DIR_ENCRYPT;
+		break;
+	case RTE_CRYPTO_CIPHER_OP_DECRYPT:
+		sess->cipher.direction = IMB_DIR_DECRYPT;
+		break;
+	default:
+		AESNI_MB_RAWDEV_LOG(ERR, "Invalid cipher operation parameter");
+		return -EINVAL;
+	}
+
+	if (xform->cipher.algo == RTE_CRYPTO_CIPHER_AES_DOCSISBPI) {
+		sess->cipher.mode = IMB_CIPHER_DOCSIS_SEC_BPI;
+	} else {
+		AESNI_MB_RAWDEV_LOG(ERR, "Unsupported cipher mode parameter");
+		return -ENOTSUP;
+	}
+
+	/* Set IV parameters */
+	sess->iv.offset = xform->cipher.iv.offset;
+	sess->iv.length = xform->cipher.iv.length;
+
+	/* Check key length and choose key expansion function for AES */
+	switch (xform->cipher.key.length) {
+	case IMB_KEY_AES_128_BYTES:
+		sess->cipher.key_length_in_bytes = IMB_KEY_AES_128_BYTES;
+		IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data,
+				sess->cipher.expanded_aes_keys.encode,
+				sess->cipher.expanded_aes_keys.decode);
+		break;
+	case IMB_KEY_AES_256_BYTES:
+		sess->cipher.key_length_in_bytes = IMB_KEY_AES_256_BYTES;
+		IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
+				sess->cipher.expanded_aes_keys.encode,
+				sess->cipher.expanded_aes_keys.decode);
+		break;
+	default:
+		AESNI_MB_RAWDEV_LOG(ERR, "Invalid cipher key length");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+aesni_mb_rawdev_pmd_enq(struct rte_rawdev *dev, struct rte_rawdev_buf **ops,
+		unsigned int nb_ops, rte_rawdev_obj_t q_id)
+{
+	unsigned int nb_enqueued;
+	struct aesni_mb_rawdev_dev *hw = dev->dev_private;
+	struct aesni_mb_rawdev_qp *qp;
+
+	qp = hw->queue_pairs[*(uint16_t *)q_id];
+
+	nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
+			(void **)ops, nb_ops, NULL);
+
+	qp->stats.enqueued_count += nb_enqueued;
+
+	return nb_enqueued;
+}
+
+static inline struct aesni_mb_rawdev_session *
+get_session(struct rte_multi_fn_op *op)
+{
+	struct aesni_mb_rawdev_session *sess = NULL;
+
+	if (likely(op->sess != NULL))
+		sess = op->sess->sess_private_data;
+	else
+		op->op_status = RTE_MULTI_FN_STATUS_INVALID_SESSION;
+
+	return sess;
+}
+
+static inline uint64_t
+err_start_offset(struct rte_multi_fn_op *op,
+		struct aesni_mb_rawdev_session *session, uint32_t oop)
+{
+	struct rte_mbuf *m_src, *m_dst;
+	uint8_t *p_src, *p_dst;
+	uintptr_t u_src, u_dst;
+	uint32_t cipher_end, err_detect_end;
+	struct rte_crypto_sym_op *s_op;
+	struct rte_multi_fn_err_detect_op *e_op;
+
+	s_op = &op->next->crypto_sym;
+	e_op = &op->err_detect;
+	m_src = op->m_src;
+	m_dst = op->m_dst;
+
+	/* Only cipher then error detect needs special calculation. */
+	if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH)
+		return op->next->err_detect.data.offset;
+
+	p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
+	p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
+	u_src = (uintptr_t)p_src;
+	u_dst = (uintptr_t)p_dst + e_op->data.offset;
+
+	/**
+	 * Copy the content between cipher offset and err detect offset
+	 * for generating correct output.
+	 */
+	if (s_op->cipher.data.offset > e_op->data.offset)
+		memcpy(p_dst + e_op->data.offset,
+				p_src + e_op->data.offset,
+				s_op->cipher.data.offset -
+				e_op->data.offset);
+
+	/**
+	 * Copy the content between (cipher offset + length) and
+	 * (error detect offset + length) for generating correct output
+	 */
+	cipher_end = s_op->cipher.data.offset + s_op->cipher.data.length;
+	err_detect_end = e_op->data.offset + e_op->data.length;
+	if (cipher_end < err_detect_end)
+		memcpy(p_dst + cipher_end, p_src + cipher_end,
+				err_detect_end - cipher_end);
+
+	/**
+	 * Since intel-ipsec-mb only supports positive values,
+	 * we need to deduct the correct offset between src and dst.
+	 */
+
+	return u_src < u_dst ? (u_dst - u_src) :
+			(UINT64_MAX - u_src + u_dst + 1);
+}
+
+static inline int
+set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_rawdev_qp *qp,
+		struct rte_multi_fn_op *op, uint8_t *output_idx)
+{
+	struct rte_mbuf *m_src, *m_dst;
+	struct rte_crypto_sym_op *s_op;
+	struct rte_multi_fn_err_detect_op *e_op;
+	uint32_t m_offset, oop;
+	struct aesni_mb_rawdev_session *session;
+
+	session = get_session(op);
+	if (session == NULL) {
+		op->op_status = RTE_MULTI_FN_STATUS_INVALID_SESSION;
+		return -1;
+	}
+
+	if (op->next == NULL) {
+		op->op_status = RTE_MULTI_FN_OP_STATUS_FAILURE;
+		return -1;
+	}
+
+	m_src = op->m_src;
+
+	/* Set crypto operation */
+	job->chain_order = session->chain_order;
+	if (session->chain_order == IMB_ORDER_CIPHER_HASH) {
+		s_op = &op->crypto_sym;
+		e_op = &op->next->err_detect;
+	} else {
+		s_op = &op->next->crypto_sym;
+		e_op = &op->err_detect;
+	}
+
+	/* Set cipher parameters */
+	job->cipher_direction = session->cipher.direction;
+	job->cipher_mode = session->cipher.mode;
+
+	job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+
+	/* Set authentication parameters */
+	job->hash_alg = session->err_detect.algo;
+
+	job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+	job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
+
+	if (!op->m_dst) {
+		/* in-place operation */
+		m_dst = m_src;
+		oop = 0;
+	} else if (op->m_dst == op->m_src) {
+		/* in-place operation */
+		m_dst = m_src;
+		oop = 0;
+	} else {
+		/* out-of-place operation */
+		m_dst = op->m_dst;
+		oop = 1;
+	}
+
+	m_offset = s_op->cipher.data.offset;
+
+	/* Set output location */
+	if (session->err_detect.operation ==
+			RTE_MULTI_FN_ERR_DETECT_OP_VERIFY) {
+		job->auth_tag_output = qp->temp_digests[*output_idx];
+		*output_idx = (*output_idx + 1) % MAX_JOBS;
+	} else {
+		job->auth_tag_output = e_op->output.data;
+	}
+
+	/* Set output length */
+	job->auth_tag_output_len_in_bytes = session->err_detect.gen_output_len;
+
+	/* Set IV parameters */
+	job->iv_len_in_bytes = session->iv.length;
+
+	/* Data Parameters */
+	job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
+	job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
+
+	job->cipher_start_src_offset_in_bytes =	s_op->cipher.data.offset;
+	job->msg_len_to_cipher_in_bytes = s_op->cipher.data.length;
+
+	job->hash_start_src_offset_in_bytes = err_start_offset(op,
+			session, oop);
+	job->msg_len_to_hash_in_bytes = e_op->data.length;
+
+	job->iv = (uint8_t *)op + session->iv.offset;
+
+	job->user_data = op;
+
+	return 0;
+}
+
+static inline void
+verify_crc(JOB_AES_HMAC *job, uint8_t *status)
+{
+	uint16_t crc_offset = job->hash_start_src_offset_in_bytes +
+				job->msg_len_to_hash_in_bytes -
+				job->cipher_start_src_offset_in_bytes;
+	uint8_t *embedded_crc = job->dst + crc_offset;
+
+	if (!job->msg_len_to_hash_in_bytes)
+		return;
+
+	/* Verify CRC (at the end of the message) */
+	if (memcmp(job->auth_tag_output, embedded_crc, 4) != 0)
+		*status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+}
+
+static inline struct rte_multi_fn_op *
+post_process_mb_job(JOB_AES_HMAC *job)
+{
+	struct rte_multi_fn_op *op = (struct rte_multi_fn_op *)
+					job->user_data;
+	struct aesni_mb_rawdev_session *sess = op->sess->sess_private_data;
+
+	AESNI_MB_RAWDEV_LOG(INFO, "struct rte_multi_fn_session %p",
+			op->sess);
+	AESNI_MB_RAWDEV_LOG(INFO, "struct aesni_mb_rawdev_session %p", sess);
+
+	if (likely(op->op_status ==
+			RTE_MULTI_FN_OP_STATUS_NOT_PROCESSED)) {
+		switch (job->status) {
+		case STS_COMPLETED:
+			op->op_status = RTE_MULTI_FN_OP_STATUS_SUCCESS;
+
+			if (job->hash_alg == IMB_AUTH_NULL)
+				break;
+
+			if (sess->err_detect.operation ==
+					RTE_MULTI_FN_ERR_DETECT_OP_VERIFY)
+				verify_crc(job, &op->op_status);
+			break;
+		default:
+			op->op_status = RTE_MULTI_FN_OP_STATUS_FAILURE;
+		}
+	}
+
+	return op;
+}
+
+static unsigned
+handle_completed_jobs(struct aesni_mb_rawdev_qp *qp, JOB_AES_HMAC *job,
+		struct rte_multi_fn_op **ops, uint16_t nb_ops)
+{
+	struct rte_multi_fn_op *op = NULL;
+	unsigned processed_jobs = 0;
+
+	while (job != NULL) {
+		op = post_process_mb_job(job);
+
+		if (op) {
+			ops[processed_jobs++] = op;
+			qp->stats.dequeued_count++;
+		} else {
+			qp->stats.dequeue_err_count++;
+			break;
+		}
+		if (processed_jobs == nb_ops)
+			break;
+
+		job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
+	}
+
+	return processed_jobs;
+}
+
+static inline uint16_t
+flush_mb_mgr(struct aesni_mb_rawdev_qp *qp, struct rte_multi_fn_op **ops,
+		uint16_t nb_ops)
+{
+	int processed_ops = 0;
+
+	/* Flush the remaining jobs */
+	JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
+
+	AESNI_MB_RAWDEV_LOG(INFO, "qp->mb_mgr %p", qp->mb_mgr);
+
+	if (job)
+		processed_ops += handle_completed_jobs(qp, job,
+				&ops[processed_ops], nb_ops - processed_ops);
+
+	return processed_ops;
+}
+
+static inline JOB_AES_HMAC *
+set_job_null_op(JOB_AES_HMAC *job, struct rte_multi_fn_op *op)
+{
+	job->chain_order = IMB_ORDER_HASH_CIPHER;
+	job->cipher_mode = IMB_CIPHER_NULL;
+	job->hash_alg = IMB_AUTH_NULL;
+	job->cipher_direction = IMB_DIR_DECRYPT;
+
+	/* Set user data to be crypto operation data struct */
+	job->user_data = op;
+
+	return job;
+}
+
+static int
+aesni_mb_rawdev_pmd_deq(struct rte_rawdev *dev, struct rte_rawdev_buf **ops,
+		unsigned int nb_ops, rte_rawdev_obj_t q_id)
+{
+	struct aesni_mb_rawdev_qp *qp;
+	struct rte_multi_fn_op *op;
+	JOB_AES_HMAC *job;
+	struct aesni_mb_rawdev_dev *hw = dev->dev_private;
+
+	qp = hw->queue_pairs[*(uint16_t *)q_id];
+	int retval;
+	unsigned int processed_jobs = 0;
+
+	if (unlikely(nb_ops == 0))
+		return 0;
+
+	AESNI_MB_RAWDEV_LOG(INFO, "qp->mb_mgr %p", qp->mb_mgr);
+
+	uint8_t output_idx = qp->output_idx;
+	do {
+		/* Get next free mb job struct from mb manager */
+		job = IMB_GET_NEXT_JOB(qp->mb_mgr);
+		if (unlikely(job == NULL)) {
+			/* if no free mb job structs we need to flush mb_mgr */
+			processed_jobs += flush_mb_mgr(qp,
+					(struct rte_multi_fn_op **)
+					&ops[processed_jobs],
+					nb_ops - processed_jobs);
+
+			if (nb_ops == processed_jobs)
+				break;
+
+			job = IMB_GET_NEXT_JOB(qp->mb_mgr);
+		}
+
+		/*
+		 * Get next operation to process from ingress queue.
+		 * There is no need to return the job to the MB_MGR
+		 * if there are no more operations to process, since the MB_MGR
+		 * can use that pointer again in next get_next calls.
+		 */
+		retval = rte_ring_dequeue(qp->ingress_queue, (void **)&op);
+		if (retval < 0)
+			break;
+
+		retval = set_mb_job_params(job, qp, op, &output_idx);
+		if (unlikely(retval != 0)) {
+			qp->stats.dequeue_err_count++;
+			set_job_null_op(job, op);
+		}
+
+		/* Submit job to multi-buffer for processing */
+#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
+		job = IMB_SUBMIT_JOB(qp->mb_mgr);
+#else
+		job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
+#endif
+		/*
+		 * If submit returns a processed job then handle it,
+		 * before submitting subsequent jobs
+		 */
+		if (job)
+			processed_jobs += handle_completed_jobs(qp, job,
+					(struct rte_multi_fn_op **)
+					&ops[processed_jobs],
+					nb_ops - processed_jobs);
+
+	} while (processed_jobs < nb_ops);
+
+	qp->output_idx = output_idx;
+
+	if (processed_jobs < 1)
+		processed_jobs += flush_mb_mgr(qp,
+				(struct rte_multi_fn_op **)
+				&ops[processed_jobs],
+				nb_ops - processed_jobs);
+
+	return processed_jobs;
+}
+
+static int
+aesni_mb_rawdev_pmd_qp_release(struct rte_rawdev *dev, uint16_t qp_id)
+{
+	struct aesni_mb_rawdev_dev *hw = dev->dev_private;
+	struct aesni_mb_rawdev_qp *qp = hw->queue_pairs[qp_id];
+	struct rte_ring *r = NULL;
+
+	if (qp != NULL) {
+		r = rte_ring_lookup(qp->name);
+		if (r)
+			rte_ring_free(r);
+		if (qp->mb_mgr)
+			free_mb_mgr(qp->mb_mgr);
+		rte_free(qp);
+		hw->queue_pairs[qp_id] = NULL;
+	}
+	return 0;
+}
+
+static int
+qp_set_unique_name(struct rte_rawdev *dev,
+		struct aesni_mb_rawdev_qp *qp)
+{
+	unsigned n = snprintf(qp->name, sizeof(qp->name),
+			"aesni_mb_pmd_%u_qp_%u",
+			dev->dev_id, qp->id);
+
+	if (n >= sizeof(qp->name))
+		return -1;
+
+	return 0;
+}
+
+static struct rte_ring *
+qp_create_processed_ops_ring(struct aesni_mb_rawdev_qp *qp,
+		unsigned int ring_size, int socket_id)
+{
+	struct rte_ring *r;
+	char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
+
+	unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
+
+	if (n >= sizeof(ring_name))
+		return NULL;
+
+	r = rte_ring_lookup(ring_name);
+	if (r) {
+		if (rte_ring_get_size(r) >= ring_size) {
+			AESNI_MB_RAWDEV_LOG(INFO, "Reusing existing ring %s for processed ops",
+			ring_name);
+			return r;
+		}
+
+		AESNI_MB_RAWDEV_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
+			ring_name);
+		return NULL;
+	}
+
+	return rte_ring_create(ring_name, ring_size, socket_id,
+			RING_F_SP_ENQ | RING_F_SC_DEQ);
+}
+
+static int
+aesni_mb_rawdev_pmd_qp_setup(struct rte_rawdev *dev, uint16_t qp_id,
+		rte_rawdev_obj_t qp_c)
+{
+
+	struct aesni_mb_rawdev_qp *qp = NULL;
+	struct aesni_mb_rawdev_dev *hw = dev->dev_private;
+	struct aesni_mb_rawdev_private *internals = &hw->priv;
+	const struct rte_multi_fn_qp_config *qp_conf =
+			(const struct rte_multi_fn_qp_config *)qp_c;
+	int ret = -1;
+
+	/* Free memory prior to re-allocation if needed. */
+	if (hw->queue_pairs[qp_id] != NULL)
+		aesni_mb_rawdev_pmd_qp_release(dev, qp_id);
+
+	/* Allocate the queue pair data structure. */
+	qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
+					RTE_CACHE_LINE_SIZE,
+					internals->socket_id);
+	if (qp == NULL)
+		return -ENOMEM;
+
+	qp->id = qp_id;
+	hw->queue_pairs[qp_id] = qp;
+
+	if (qp_set_unique_name(dev, qp))
+		goto qp_setup_cleanup;
+
+	qp->mb_mgr = alloc_mb_mgr(0);
+	AESNI_MB_RAWDEV_LOG(INFO, "mb_mgr %p", qp->mb_mgr);
+	if (qp->mb_mgr == NULL) {
+		ret = -ENOMEM;
+		goto qp_setup_cleanup;
+	}
+
+	switch (internals->vector_mode) {
+	case AESNI_MB_RAWDEV_SSE:
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
+		init_mb_mgr_sse(qp->mb_mgr);
+		break;
+	case AESNI_MB_RAWDEV_AVX:
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
+		init_mb_mgr_avx(qp->mb_mgr);
+		break;
+	case AESNI_MB_RAWDEV_AVX2:
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
+		init_mb_mgr_avx2(qp->mb_mgr);
+		break;
+	case AESNI_MB_RAWDEV_AVX512:
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
+		init_mb_mgr_avx512(qp->mb_mgr);
+		break;
+	default:
+		AESNI_MB_RAWDEV_LOG(ERR, "Unsupported vector mode %u\n",
+				internals->vector_mode);
+		goto qp_setup_cleanup;
+	}
+
+	qp->ingress_queue = qp_create_processed_ops_ring(qp,
+			qp_conf->nb_descriptors, rte_socket_id());
+	if (qp->ingress_queue == NULL) {
+		ret = -1;
+		goto qp_setup_cleanup;
+	}
+
+	memset(&qp->stats, 0, sizeof(qp->stats));
+
+	char mp_name[RTE_MEMPOOL_NAMESIZE];
+
+	snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
+				"digest_mp_%u_%u", dev->dev_id, qp_id);
+	return 0;
+
+qp_setup_cleanup:
+	if (qp) {
+		if (qp->mb_mgr)
+			free_mb_mgr(qp->mb_mgr);
+		rte_free(qp);
+	}
+
+	return ret;
+}
+
+static uint16_t
+aesni_mb_rawdev_pmd_qp_count(struct rte_rawdev *dev)
+{
+	struct aesni_mb_rawdev_dev *hw = dev->dev_private;
+	return hw->num_queue_pair;
+}
+
+static enum aesni_mb_rawdev_op
+session_support(struct rte_multi_fn_xform *xform)
+{
+	struct rte_crypto_sym_xform *crypto_sym;
+	struct rte_multi_fn_err_detect_xform *err_detect;
+	struct rte_multi_fn_xform *next;
+	enum aesni_mb_rawdev_op op = AESNI_MB_RAWDEV_OP_NOT_SUPPORTED;
+
+	next = xform->next;
+	/* err detect generate -> cipher encrypt */
+	if (xform->type == RTE_MULTI_FN_XFORM_TYPE_ERR_DETECT) {
+
+		err_detect = &xform->err_detect;
+		if ((err_detect->algo ==
+				RTE_MULTI_FN_ERR_DETECT_CRC32_ETH) &&
+		    (err_detect->op ==
+				RTE_MULTI_FN_ERR_DETECT_OP_GENERATE) &&
+		     next != NULL &&
+		     next->type == RTE_MULTI_FN_XFORM_TYPE_CRYPTO_SYM) {
+
+			crypto_sym = &next->crypto_sym;
+			if (crypto_sym->type ==
+				RTE_CRYPTO_SYM_XFORM_CIPHER &&
+			    crypto_sym->cipher.op ==
+				RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
+			    crypto_sym->cipher.algo ==
+				RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
+			    crypto_sym->cipher.key.length ==
+				IMB_KEY_AES_128_BYTES &&
+			    crypto_sym->cipher.iv.length ==
+				AES_BLOCK_SIZE) {
+				op = AESNI_MB_RAWDEV_OP_ERR_DETECT_CIPHER;
+			}
+		}
+	/* cipher decrypt -> err detect verify */
+	} else if (xform->type == RTE_MULTI_FN_XFORM_TYPE_CRYPTO_SYM) {
+		crypto_sym = &xform->crypto_sym;
+		if (crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+		    crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
+		    crypto_sym->cipher.algo ==
+				    RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
+		    crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES &&
+		    crypto_sym->cipher.iv.length == AES_BLOCK_SIZE &&
+		    next != NULL &&
+		    next->type == RTE_MULTI_FN_XFORM_TYPE_ERR_DETECT) {
+
+			err_detect = &next->err_detect;
+			if ((err_detect->algo ==
+				RTE_MULTI_FN_ERR_DETECT_CRC32_ETH) &&
+			    (err_detect->op ==
+				RTE_MULTI_FN_ERR_DETECT_OP_VERIFY)) {
+				op = AESNI_MB_RAWDEV_OP_CIPHER_ERR_DETECT;
+			}
+		}
+
+	}
+
+	return op;
+}
+
+static struct rte_multi_fn_session *
+aesni_mb_pmd_session_create(struct rte_rawdev *dev,
+		struct rte_multi_fn_xform *xform,
+		int socket_id)
+{
+	int ret;
+	enum aesni_mb_rawdev_op op;
+	struct rte_multi_fn_err_detect_xform *err_detect_xform;
+	struct rte_crypto_sym_xform *cipher_xform;
+	struct rte_multi_fn_session *session;
+	struct aesni_mb_rawdev_session *aesni_sess = NULL;
+	struct aesni_mb_rawdev_dev *d = dev->dev_private;
+	struct aesni_mb_rawdev_private *internals = &d->priv;
+
+	op = session_support(xform);
+
+	/* Alloc rte_multi_fn_session */
+	session = rte_zmalloc_socket("rte_multi_fn_session",
+				sizeof(struct rte_multi_fn_session),
+				RTE_CACHE_LINE_MIN_SIZE,
+				socket_id);
+
+	if (!session) {
+		AESNI_MB_RAWDEV_LOG(ERR, "rte_multi_fn_session alloc failed");
+		return NULL;
+	}
+
+	/* Alloc aesni_mb_rawdev_session */
+	aesni_sess = rte_zmalloc_socket("aesni_mb_rawdev_session",
+				sizeof(struct aesni_mb_rawdev_session),
+				RTE_CACHE_LINE_MIN_SIZE,
+				socket_id);
+
+	if (!aesni_sess) {
+		AESNI_MB_RAWDEV_LOG(ERR, "aesni_mb_rawdev_session alloc failed");
+		return NULL;
+	}
+
+	session->sess_private_data = aesni_sess;
+
+	switch (op) {
+	case AESNI_MB_RAWDEV_OP_ERR_DETECT_CIPHER:
+		aesni_sess->chain_order = IMB_ORDER_HASH_CIPHER;
+		err_detect_xform = &xform->err_detect;
+		cipher_xform = &xform->next->crypto_sym;
+		break;
+	case AESNI_MB_RAWDEV_OP_CIPHER_ERR_DETECT:
+		aesni_sess->chain_order = IMB_ORDER_CIPHER_HASH;
+		cipher_xform = &xform->crypto_sym;
+		err_detect_xform = &xform->next->err_detect;
+		break;
+	default:
+		AESNI_MB_RAWDEV_LOG(ERR, "Unsupported multi-function xform chain");
+		return NULL;
+	}
+
+	/* Default IV length = 0 */
+	aesni_sess->iv.length = 0;
+
+	ret = set_session_err_detect_parameters(aesni_sess,
+			err_detect_xform);
+
+	if (ret != 0) {
+		AESNI_MB_RAWDEV_LOG(ERR, "Invalid/unsupported error detect parameters");
+		return NULL;
+	}
+
+	ret = set_session_cipher_parameters(internals->mb_mgr,
+			aesni_sess,
+			cipher_xform);
+
+	if (ret != 0) {
+		AESNI_MB_RAWDEV_LOG(ERR, "Invalid/unsupported cipher parameters");
+		return NULL;
+	}
+
+	return session;
+}
+
+static int
+aesni_mb_rawdev_pmd_session_destroy(struct rte_rawdev *dev __rte_unused,
+				 struct rte_multi_fn_session *sess)
+{
+
+	if (sess) {
+		if (sess->sess_private_data)
+			rte_free(sess->sess_private_data);
+		rte_free(sess);
+	}
+
+	return 0;
+}
+
+#define MAX_QUEUES 64
+
+static void
+aesni_mb_rawdev_pmd_info_get(struct rte_rawdev *dev __rte_unused,
+				  rte_rawdev_obj_t dev_info)
+{
+	struct rte_multi_fn_dev_config *config = NULL;
+	struct rte_multi_fn_device_info *dev_priv = dev_info;
+	dev_priv->create = aesni_mb_pmd_session_create;
+	dev_priv->destroy = aesni_mb_rawdev_pmd_session_destroy;
+	config = dev_priv->config;
+	config->nb_queues = MAX_QUEUES;
+}
+
+int
+pmd_parse_uint_arg(const char *key __rte_unused,
+		const char *value, void *extra_args)
+{
+	int i;
+	char *end;
+	errno = 0;
+
+	i = strtol(value, &end, 10);
+	if (*end != 0 || errno != 0 || i < 0)
+		return -EINVAL;
+
+	*((uint32_t *)extra_args) = i;
+	return 0;
+}
+
+int
+parse_input_args(unsigned int *socket_id,
+		const char *args)
+{
+	struct rte_kvargs *kvlist = NULL;
+	int ret = 0;
+
+	if (socket_id == NULL)
+		return -EINVAL;
+
+	if (args) {
+		kvlist = rte_kvargs_parse(args,	rawdev_pmd_valid_params);
+		if (kvlist == NULL)
+			return -EINVAL;
+
+		ret = rte_kvargs_process(kvlist,
+				AESNI_MB_RAWDEV_PMD_SOCKET_ID_ARG,
+				&pmd_parse_uint_arg,
+				socket_id);
+		if (ret < 0)
+			goto free_kvlist;
+
+	}
+
+free_kvlist:
+	rte_kvargs_free(kvlist);
+	return ret;
+}
+
+#define RTE_RAWDEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 8
+
+static const char * const xstat_names[] = {
+		"successful_enqueues", "successful_dequeues",
+		"failed_enqueues", "failed_dequeues",
+};
+
+static int
+aesni_mb_rawdev_xstats_get(const struct rte_rawdev *dev,
+		const unsigned int ids[],
+		uint64_t values[], unsigned int n)
+{
+	unsigned int i;
+	struct aesni_mb_rawdev_dev *hw = dev->dev_private;
+	struct aesni_mb_rawdev_qp *qp;
+	struct aesni_mb_rawdev_stats stats = {0};
+	int qp_id;
+
+	for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) {
+		qp = hw->queue_pairs[qp_id];
+
+		stats.enqueued_count += qp->stats.enqueued_count;
+		stats.dequeued_count += qp->stats.dequeued_count;
+
+		stats.enqueue_err_count += qp->stats.enqueue_err_count;
+		stats.dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+
+	for (i = 0; i < n; i++) {
+		switch (ids[i]) {
+		case 0:
+			values[i] = stats.enqueued_count;
+			break;
+		case 1:
+			values[i] = stats.dequeued_count;
+			break;
+		case 2:
+			values[i] = stats.enqueue_err_count;
+			break;
+		case 3:
+			values[i] = stats.dequeue_err_count;
+			break;
+		default:
+			values[i] = 0;
+			break;
+		}
+	}
+	return n;
+}
+
+static int
+aesni_mb_rawdev_xstats_get_names(const struct rte_rawdev *dev,
+		struct rte_rawdev_xstats_name *names,
+		unsigned int size)
+{
+	unsigned int i;
+
+	RTE_SET_USED(dev);
+	if (size < RTE_DIM(xstat_names))
+		return RTE_DIM(xstat_names);
+
+	for (i = 0; i < RTE_DIM(xstat_names); i++)
+		strlcpy(names[i].name, xstat_names[i], sizeof(names[i]));
+
+	return RTE_DIM(xstat_names);
+}
+
+static int
+aesni_mb_rawdev_xstats_reset(struct rte_rawdev *dev, const uint32_t *ids,
+		uint32_t nb_ids)
+{
+	struct aesni_mb_rawdev_dev *hw = dev->dev_private;
+	struct aesni_mb_rawdev_qp *qp;
+	unsigned int i;
+	int qp_id;
+
+	if (!ids) {
+
+		for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) {
+			qp = hw->queue_pairs[qp_id];
+			qp->stats.enqueued_count = 0;
+			qp->stats.dequeued_count = 0;
+			qp->stats.enqueue_err_count = 0;
+			qp->stats.dequeue_err_count = 0;
+		}
+		return 0;
+	}
+
+	for (i = 0; i < nb_ids; i++) {
+		switch (ids[i]) {
+		case 0:
+			for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) {
+				qp = hw->queue_pairs[qp_id];
+				qp->stats.enqueued_count = 0;
+			}
+			break;
+		case 1:
+			for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) {
+				qp = hw->queue_pairs[qp_id];
+				qp->stats.dequeued_count = 0;
+			}
+			break;
+		case 2:
+			for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) {
+				qp = hw->queue_pairs[qp_id];
+				qp->stats.enqueue_err_count = 0;
+			}
+			break;
+		case 3:
+			for (qp_id = 0; qp_id < hw->num_queue_pair; qp_id++) {
+				qp = hw->queue_pairs[qp_id];
+				qp->stats.dequeue_err_count = 0;
+			}
+			break;
+		default:
+			AESNI_MB_RAWDEV_LOG(ERR,
+					"Invalid xstat id - cannot reset val");
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int
+rawdev_aesni_mb_create(struct rte_rawdev *dev,
+		struct rte_multi_fn_dev_config *init_params)
+{
+
+	static const struct rte_rawdev_ops aesni_mb_raw_ops = {
+		.enqueue_bufs = aesni_mb_rawdev_pmd_enq,
+		.dequeue_bufs = aesni_mb_rawdev_pmd_deq,
+		.dev_configure = aesni_mb_rawdev_pmd_config,
+		.dev_info_get = aesni_mb_rawdev_pmd_info_get,
+		.dev_start = aesni_mb_rawdev_pmd_start,
+		.dev_stop = aesni_mb_rawdev_pmd_stop,
+		.dev_close = aesni_mb_rawdev_pmd_close,
+		.queue_setup = aesni_mb_rawdev_pmd_qp_setup,
+		.queue_release = aesni_mb_rawdev_pmd_qp_release,
+		.queue_count = aesni_mb_rawdev_pmd_qp_count,
+		.xstats_get = aesni_mb_rawdev_xstats_get,
+		.xstats_get_names = aesni_mb_rawdev_xstats_get_names,
+		.xstats_reset = aesni_mb_rawdev_xstats_reset,
+	};
+
+	struct aesni_mb_rawdev_dev *hw;
+	struct aesni_mb_rawdev_private *internals;
+	enum aesni_mb_rawdev_vector_mode vector_mode;
+	MB_MGR *mb_mgr;
+
+	hw = dev->dev_private;
+
+	/* Check CPU for supported vector instruction set */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
+		vector_mode = AESNI_MB_RAWDEV_AVX512;
+	else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
+		vector_mode = AESNI_MB_RAWDEV_AVX2;
+	else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
+		vector_mode = AESNI_MB_RAWDEV_AVX;
+	else
+		vector_mode = AESNI_MB_RAWDEV_SSE;
+
+	dev->dev_ops = &aesni_mb_raw_ops;
+
+	hw->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+
+	/* Check CPU for support for AES instruction set */
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
+	else
+		AESNI_MB_RAWDEV_LOG(WARNING, "AES instructions not supported by CPU");
+
+	mb_mgr = alloc_mb_mgr(0);
+	AESNI_MB_RAWDEV_LOG(INFO, "create() mb_mgr %p", mb_mgr);
+
+	if (mb_mgr == NULL)
+		return -ENOMEM;
+
+	switch (vector_mode) {
+	case AESNI_MB_RAWDEV_SSE:
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
+		init_mb_mgr_sse(mb_mgr);
+		break;
+	case AESNI_MB_RAWDEV_AVX:
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
+		init_mb_mgr_avx(mb_mgr);
+		break;
+	case AESNI_MB_RAWDEV_AVX2:
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
+		init_mb_mgr_avx2(mb_mgr);
+		break;
+	case AESNI_MB_RAWDEV_AVX512:
+		hw->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
+		init_mb_mgr_avx512(mb_mgr);
+		break;
+	default:
+		AESNI_MB_RAWDEV_LOG(ERR, "Unsupported vector mode %u\n",
+			vector_mode);
+		goto error_exit;
+	}
+
+	/* Set vector instructions mode supported */
+	internals = &hw->priv;
+
+	internals->vector_mode = vector_mode;
+	internals->max_nb_queue_pairs = init_params->nb_queues;
+	internals->mb_mgr = mb_mgr;
+	internals->socket_id = init_params->socket_id;
+
+	AESNI_MB_RAWDEV_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
+			imb_get_version_str());
+
+	return 0;
+
+error_exit:
+	if (mb_mgr)
+		free_mb_mgr(mb_mgr);
+
+	rte_rawdev_pmd_release(dev);
+
+	return -1;
+}
+
+static int
+aesni_mb_rawdev_probe(struct rte_vdev_device *vdev)
+{
+	const char *vdev_name;
+	const char *args;
+	struct rte_rawdev *rawdev;
+	struct rte_multi_fn_dev_config init_params = {
+			.nb_queues = 64,
+			.socket_id = SOCKET_ID_ANY};
+
+	vdev_name = rte_vdev_device_name(vdev);
+	if (vdev_name == NULL)
+		return -EINVAL;
+
+	args = rte_vdev_device_args(vdev);
+
+	/* Parse args */
+	parse_input_args(&init_params.socket_id, args);
+
+	AESNI_MB_RAWDEV_LOG(INFO, "device name %s", vdev_name);
+
+	/* Allocate device structure */
+	rawdev = rte_rawdev_pmd_allocate(vdev_name,
+			sizeof(struct aesni_mb_rawdev_dev),
+			rte_socket_id());
+
+	if (!rawdev) {
+		AESNI_MB_RAWDEV_LOG(ERR, "Unable to allocate raw device");
+		return -EINVAL;
+	}
+
+	rawdev->driver_name = RTE_STR(rawdev_aesni_mb);
+
+	AESNI_MB_RAWDEV_LOG(INFO, "Driver name %s", rawdev->driver_name);
+
+	return rawdev_aesni_mb_create(rawdev, &init_params);
+}
+
+static int
+aesni_mb_rawdev_remove(struct rte_vdev_device *vdev)
+{
+	struct rte_rawdev *dev;
+	struct aesni_mb_rawdev_dev *hw;
+	struct aesni_mb_rawdev_private *internals;
+	const char *name;
+
+	name = rte_vdev_device_name(vdev);
+	if (name == NULL) {
+		AESNI_MB_RAWDEV_LOG(INFO, "Driver name %s", name);
+		return -EINVAL;
+	}
+
+	dev = rte_rawdev_pmd_get_named_dev(name);
+	if (dev == NULL)
+		return -ENODEV;
+
+	hw = dev->dev_private;
+	internals = &hw->priv;
+
+	free_mb_mgr(internals->mb_mgr);
+
+	return rte_rawdev_pmd_release(dev);
+}
+
+static struct rte_vdev_driver rawdev_aesni_mb_pmd_drv = {
+	.probe = aesni_mb_rawdev_probe,
+	.remove = aesni_mb_rawdev_remove
+};
+
+RTE_PMD_REGISTER_VDEV(rawdev_aesni_mb, rawdev_aesni_mb_pmd_drv);
+
+RTE_INIT(aesni_mb_raw_init_log)
+{
+	aesni_mb_rawdev_logtype_driver = rte_log_register("pmd.raw.aesni_mb");
+}
diff --git a/drivers/raw/aesni_mb/aesni_mb_rawdev_private.h b/drivers/raw/aesni_mb/aesni_mb_rawdev_private.h
new file mode 100644
index 0000000..12a3f16
--- /dev/null
+++ b/drivers/raw/aesni_mb/aesni_mb_rawdev_private.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#ifndef _AESNI_MB_RAWDEV_PRIVATE_H_
+#define _AESNI_MB_RAWDEV_PRIVATE_H_
+
+#include <intel-ipsec-mb.h>
+#include <rte_rawdev.h>
+#include <rte_multi_fn.h>
+
+enum aesni_mb_rawdev_vector_mode {
+	AESNI_MB_RAWDEV_NOT_SUPPORTED = 0,
+	AESNI_MB_RAWDEV_SSE,
+	AESNI_MB_RAWDEV_AVX,
+	AESNI_MB_RAWDEV_AVX2,
+	AESNI_MB_RAWDEV_AVX512
+};
+
+#define AESNI_MB_RAWDEV_PMD_SOCKET_ID_ARG			("socket_id")
+
+/* AESNI_MB PMD LOGTYPE DRIVER */
+int aesni_mb_rawdev_logtype_driver;
+
+#define AESNI_MB_RAWDEV_LOG(level, fmt, ...)  \
+	rte_log(RTE_LOG_ ## level, aesni_mb_rawdev_logtype_driver,  \
+			"%s() line %u: " fmt "\n", __func__, __LINE__,  \
+					## __VA_ARGS__)
+
+/* Maximum length for digest */
+#define DIGEST_LENGTH_MAX 4
+
+/* AESNI-MB operation order */
+enum aesni_mb_rawdev_op {
+	AESNI_MB_RAWDEV_OP_ERR_DETECT_CIPHER,
+	AESNI_MB_RAWDEV_OP_CIPHER_ERR_DETECT,
+	AESNI_MB_RAWDEV_OP_NOT_SUPPORTED
+};
+
+static const unsigned auth_digest_byte_lengths[] = {
+		[IMB_AUTH_DOCSIS_CRC32]  = 4,
+       /**< Vector mode dependent pointer table of the multi-buffer APIs */
+};
+
+/**
+ * Get the full digest size in bytes for a specified authentication algorithm
+ * (if available in the Multi-buffer library)
+ *
+ * @Note: this function will not return a valid value for a non-valid
+ * authentication algorithm
+ */
+static inline unsigned
+get_digest_byte_length(JOB_HASH_ALG algo)
+{
+	return auth_digest_byte_lengths[algo];
+}
+
+/* AESNI-MB device statistics */
+struct aesni_mb_rawdev_stats {
+	uint64_t enqueued_count;
+	uint64_t dequeued_count;
+	uint64_t enqueue_err_count;
+	uint64_t dequeue_err_count;
+};
+
+/* Private data structure for each virtual AESNI-MB device */
+struct aesni_mb_rawdev_private {
+	enum aesni_mb_rawdev_vector_mode vector_mode;
+	unsigned max_nb_queue_pairs;
+	MB_MGR *mb_mgr;
+	unsigned int socket_id;
+};
+
+/* AESNI-MB queue pair */
+struct aesni_mb_rawdev_qp {
+	uint16_t id;
+	char name[RTE_RAWDEV_NAME_MAX_LEN];
+	MB_MGR *mb_mgr;
+	struct rte_ring *ingress_queue;
+	struct aesni_mb_rawdev_stats stats;
+	uint8_t output_idx;
+	uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];
+} __rte_cache_aligned;
+
+/* AESNI-MB device data */
+struct aesni_mb_rawdev_dev {
+	struct aesni_mb_rawdev_private priv;
+	uint16_t num_queue_pair;
+	struct aesni_mb_rawdev_qp **queue_pairs;
+	uint64_t feature_flags;
+};
+
+/* AESNI-MB private session structure */
+struct aesni_mb_rawdev_session {
+	JOB_CHAIN_ORDER chain_order;
+	struct {
+		uint16_t length;
+		uint16_t offset;
+	} iv;
+	struct {
+		JOB_CIPHER_DIRECTION direction;
+		JOB_CIPHER_MODE mode;
+
+		uint64_t key_length_in_bytes;
+
+		union {
+			struct {
+				uint32_t encode[60] __rte_aligned(16);
+				uint32_t decode[60] __rte_aligned(16);
+			} expanded_aes_keys;
+		};
+	} cipher;
+	struct {
+		JOB_HASH_ALG algo;
+		enum rte_multi_fn_err_detect_operation operation;
+		uint16_t gen_output_len;
+
+	} err_detect;
+} __rte_cache_aligned;
+
+#endif /* _AESNI_MB_RAWDEV_PRIVATE_H_ */
diff --git a/drivers/raw/aesni_mb/meson.build b/drivers/raw/aesni_mb/meson.build
new file mode 100644
index 0000000..3e16874
--- /dev/null
+++ b/drivers/raw/aesni_mb/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+IMB_required_ver = '0.52.0'
+lib = cc.find_library('IPSec_MB', required: false)
+if not lib.found()
+	build = false
+	reason = 'missing dependency, "libIPSec_MB"'
+else
+	ext_deps += lib
+
+	# version comes with quotes, so we split based on " and take the middle
+	imb_ver = cc.get_define('IMB_VERSION_STR',
+		prefix : '#include<intel-ipsec-mb.h>').split('"')[1]
+
+	if (imb_ver == '') or (imb_ver.version_compare('<' + IMB_required_ver))
+		reason = 'IPSec_MB version >= @0@ is required, found version @1@'.format(
+				IMB_required_ver, imb_ver)
+		build = false
+	endif
+
+endif
+
+sources = files('rte_aesni_mb_pmd.c', 'rte_aesni_mb_pmd_ops.c')
+allow_experimental_apis = true
+deps += ['bus_vdev']
diff --git a/drivers/raw/aesni_mb/rte_pmd_rawdev_aesni_mb_version.map b/drivers/raw/aesni_mb/rte_pmd_rawdev_aesni_mb_version.map
new file mode 100644
index 0000000..fa9e17c
--- /dev/null
+++ b/drivers/raw/aesni_mb/rte_pmd_rawdev_aesni_mb_version.map
@@ -0,0 +1,3 @@
+DPDK_20.0 {
+	local: *;
+};
\ No newline at end of file
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index d295ca0..6740cc0 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -347,6 +347,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV)   += -lrte_rawdev_ioat
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += -lrte_rawdev_ntb
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV) += -lrte_rawdev_octeontx2_dma
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_EP_RAWDEV) += -lrte_rawdev_octeontx2_ep
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_AESNI_MB),y)
+ifeq ($(CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV),y)
+_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV) += -lrte_pmd_aesni_mb_rawdev
+endif # CONFIG_RTE_LIBRTE_PMD_AESNI_MB
+endif # CONFIG_RTE_LIBRTE_PMD_AESNI_MB_RAWDEV
 endif # CONFIG_RTE_LIBRTE_RAWDEV
 
 endif # !CONFIG_RTE_BUILD_SHARED_LIBS
-- 
1.7.0.7