[dpdk-dev] [PATCH v2] examples: add multi process crypto application

[Arek Kusztal <arkadiuszx.kusztal@intel.com>]

This patch adds example application that can test
usage of cryptodev in multi process environment.
More can be found in mp_crypto.rst in sample app guides.

Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
---
 doc/guides/sample_app_ug/index.rst     |    1 +
 doc/guides/sample_app_ug/mp_crypto.rst |  153 +++++
 examples/mp_crypto/Makefile            |   58 ++
 examples/mp_crypto/main.c              | 1109 ++++++++++++++++++++++++++++++++
 examples/mp_crypto/meson.build         |   18 +
 examples/mp_crypto/mp_crypto.c         |  139 ++++
 examples/mp_crypto/mp_crypto.h         |  224 +++++++
 examples/mp_crypto/mp_crypto_ipc.c     |   32 +
 examples/mp_crypto/mp_crypto_parser.c  |  511 +++++++++++++++
 examples/mp_crypto/mp_crypto_parser.h  |  149 +++++
 examples/mp_crypto/mp_crypto_vectors.c |  175 +++++
 examples/mp_crypto/mp_crypto_vectors.h |   66 ++
 12 files changed, 2635 insertions(+)
 create mode 100644 doc/guides/sample_app_ug/mp_crypto.rst
 create mode 100644 examples/mp_crypto/Makefile
 create mode 100644 examples/mp_crypto/main.c
 create mode 100644 examples/mp_crypto/meson.build
 create mode 100644 examples/mp_crypto/mp_crypto.c
 create mode 100644 examples/mp_crypto/mp_crypto.h
 create mode 100644 examples/mp_crypto/mp_crypto_ipc.c
 create mode 100644 examples/mp_crypto/mp_crypto_parser.c
 create mode 100644 examples/mp_crypto/mp_crypto_parser.h
 create mode 100644 examples/mp_crypto/mp_crypto_vectors.c
 create mode 100644 examples/mp_crypto/mp_crypto_vectors.h

diff --git a/doc/guides/sample_app_ug/index.rst b/doc/guides/sample_app_ug/index.rst
index affa9c5..ff033e4 100644
--- a/doc/guides/sample_app_ug/index.rst
+++ b/doc/guides/sample_app_ug/index.rst
@@ -35,6 +35,7 @@ Sample Applications User Guides
     link_status_intr
     server_node_efd
     service_cores
+    mp_crypto
     multi_process
     qos_metering
     qos_scheduler
diff --git a/doc/guides/sample_app_ug/mp_crypto.rst b/doc/guides/sample_app_ug/mp_crypto.rst
new file mode 100644
index 0000000..d3cb1d9
--- /dev/null
+++ b/doc/guides/sample_app_ug/mp_crypto.rst
@@ -0,0 +1,153 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+.. _mp_crypto:
+
+Multi-process Crypto Sample Application
+=======================================
+
+The Multi-process Crypto application is a simple application that
+allows to run crypto related operations in a multiple process environment. It
+builds on the EAL primary/secondary process infrastructure.
+
+The application allows a user to configure devices, setup queue-pairs, create
+and init sessions and specify data-path flow (enqueue/dequeue) in different
+processes. The app can help to check if the PMD behaves correctly
+in scenarios like the following:
+
+* device is configured in primary process, queue-pairs are setup in secondary process
+
+* queue pair is shared across processes, i.e. enqueue in one process and dequeue in another
+
+
+Compiling the Application
+-------------------------
+
+To compile the sample application see :doc:`compiling`.
+
+The application is located in the ``mp_crypto`` sub-directory.
+
+Running the Application
+-----------------------
+
+App binary: mp_crypto (in mp_crypto/build/app)
+
+For running PRIMARY or SECONDARY process standard EAL options apply:
+
+.. code-block:: console
+
+    ./mp_crypto --proc-type primary
+
+    ./mp_crypto --proc-type secondary
+
+.. Note::
+
+	The same set of BDFs must be passed to all processes.
+
+.. Note::
+	The same crypto devices must be created in all processes, e.g. in qat
+	case if asym and sym devices are enabled in the primary process, they
+	must be enabled in all secondary processes.
+
+General help can by checked by running:
+
+.. code-block:: console
+
+    ./mp_crypto -- -h
+
+The application has a number of command line options:
+
+.. code-block:: console
+
+    ./mp_crypto -- --devtype [dev-name]
+
+This option specifies which driver to use by its name (for example "crypto_qat").
+The same name must be passed to all processes.
+
+.. code-block:: console
+
+    ./mp_crypto -- --config_dev [devA, devB,]
+
+This option specifies the list of devices that should be configured by this process,
+this results in a call to the ``rte_cryptodev_configure`` API. devX is a positive
+integer (including zero), the value is according to probe order (from the smallest
+BDF number), not necessarily the cmdline order.
+
+Example command:
+
+.. code-block:: console
+
+    ./mp_crypto -w 03:01.2 -w 03:01.1 -w 03:01.3 --config-dev 0,2
+
+will configure devices 03:01.1 and 03:01.3.
+
+.. code-block:: console
+
+    ./mp_crypto -- --qp-config=[devA]:[qp_A, qp_B,];[devB]:[qp_A, qp_C];
+
+devX - positive integer (including zero), as in config_dev command
+
+qp_X - positive integer (including zero), specifies which queue pair should be setup
+
+This command specifies which queue pairs should be setup, resulting in a call to
+``rte_cryptodev_queue_pair_setup`` API.
+
+.. code-block:: console
+
+    ./mp_crypto -w 03:01.2 -w 03:01.1 -w 03:01.3 --qp-config="0:0,1;1:1;2:0,1;"
+
+This command will configure queue pairs 0 and 1 on device 0 (03:01.1), queue pair 1
+on device 1 (03:01.2), queue pairs 0 and 1 on device 2 (03:01.3). The device in question
+should be configured before that, though not necessarily by the same process.
+
+.. code-block:: console
+
+    ./mp_crypto -- --enq=[devX]:[qpX]:[ops]:[vector_id]
+    ./mp_crypto -- --deq=[devX]:[qpX]:[ops]:[vector_id]
+
+devX - positive integer (including zero), as in config_dev command
+
+qp_X - positive integer (including zero), as in qp-config command
+
+ops - when positive integer - number of operations to enqueue/dequeue, when 0 infinite loop
+
+vector_id - positive integer (including zero), vector_id used by this process
+
+This commands will enqueue/dequeue "ops" number of packets to qp_X on devX.
+Example usage:
+
+.. code-block:: console
+
+    ./mp_crypto -- --enq=2:0:0:0, --deq=2:0:0:0,
+
+Note. ',' comma character is necessary at the end due to some parser shortcomings.
+
+To close the application when running in an infinite loop a signal handler is
+registered to catch interrupt signals i.e. ``ctrl-c`` should be used. When
+used in primary process other processes will be notified about exiting
+intention and will close after collecting remaining packets (if dequeuing).
+
+Example commands
+----------------
+
+Use two different devices on 3 separate queues:
+
+.. code-block:: console
+
+    ./mp_crypto --proc-type primary -c 1 -w 03:01.1 -w 03:01.2 -- --devtype "crypto_qat" --config-dev 0,1   --qp-config="0:0,1;1:0,1;" --session-mask=0x3  --enq=0:0:0:0, --deq=0:0:0:0,  --print-stats
+    ./mp_crypto --proc-type secondary -c 2 -w 03:01.1 -w 03:01.2 -- --devtype "crypto_qat"  --enq=0:1:0:0, --deq=0:1:0:0,  --print-stats
+    ./mp_crypto --proc-type secondary -c 4 -w 03:01.1 -w 03:01.2 -- --devtype "crypto_qat"  --enq=1:0:0:0, --deq=1:0:0:0,  --print-stats
+
+Use different processes to enqueue and dequeue to one queue pair:
+
+.. code-block:: console
+
+    ./mp_crypto --proc-type primary -c 1 -w 03:01.1 -- --devtype "crypto_qat" --config-dev 0    --session-mask=0x3 --qp-config="0:1;"   --enq=0:1:0:0,   --print-stats
+    ./mp_crypto --proc-type secondary -c 2 -w 03:01.1 -- --devtype "crypto_qat"  --deq=0:1:0:0,   --print-stats
+
+Limitations
+-----------
+
+Software devices are not supported currently, but small changes in code suffice to enable it.
+
+Only one crypto vector and session type is possible to chose right now and it is AES-GCM test case.
diff --git a/examples/mp_crypto/Makefile b/examples/mp_crypto/Makefile
new file mode 100644
index 0000000..4c75446
--- /dev/null
+++ b/examples/mp_crypto/Makefile
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2020 Intel Corporation
+
+# binary name
+APP = mp_crypto
+
+# all source are stored in SRCS-y
+SRCS-y := main.c mp_crypto_parser.c mp_crypto.c mp_crypto_ipc.c mp_crypto_vectors.c
+
+# Build using pkg-config variables if possible
+ifeq ($(shell pkg-config --exists libdpdk && echo 0),0)
+
+all: shared
+.PHONY: shared static
+shared: build/$(APP)-shared
+	ln -sf $(APP)-shared build/$(APP)
+static: build/$(APP)-static
+	ln -sf $(APP)-static build/$(APP)
+
+PKGCONF ?= pkg-config
+
+PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null)
+CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk)
+LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk)
+LDFLAGS_STATIC = -Wl,-Bstatic $(shell $(PKGCONF) --static --libs libdpdk)
+
+build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
+	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED)
+
+build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build
+	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC)
+
+build:
+	@mkdir -p $@
+
+.PHONY: clean
+clean:
+	rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared
+	test -d build && rmdir -p build || true
+
+else # Build using legacy build system
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, detect a build directory, by looking for a path with a .config
+RTE_TARGET ?= $(notdir $(abspath $(dir $(firstword $(wildcard $(RTE_SDK)/*/.config)))))
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+CFLAGS += -I$(SRCDIR)
+CFLAGS += -O3 $(USER_FLAGS)
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+endif
diff --git a/examples/mp_crypto/main.c b/examples/mp_crypto/main.c
new file mode 100644
index 0000000..238b80d
--- /dev/null
+++ b/examples/mp_crypto/main.c
@@ -0,0 +1,1109 @@
+#include <rte_hexdump.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev.h>
+#include <cmdline_parse.h>
+#include <rte_cycles.h>
+#include <rte_atomic.h>
+#include <signal.h>
+
+#include "mp_crypto_parser.h"
+#include "mp_crypto.h"
+
+int
+mp_crypto_init_devs(void) {
+	uint8_t valid_devs[RTE_CRYPTO_MAX_DEVS];
+	struct rte_cryptodev_config conf;
+	struct rte_cryptodev_info info;
+	int nb_devs = 0;
+	int i;
+
+	for (i = 0; i < RTE_CRYPTO_MAX_DEVS; i++)
+		mp_app_devs[i].id = -1;
+
+	if (mp_app_driver_id == -1) {
+		MP_APP_LOG(ERR, COL_RED, "No driver of type %s registered",
+				mp_app_params->devtype_name);
+		return -1;
+	}
+
+	nb_devs = rte_cryptodev_devices_get(mp_app_params->devtype_name,
+					valid_devs, RTE_CRYPTO_MAX_DEVS);
+
+	if (nb_devs < 1) {
+		MP_APP_LOG(ERR, COL_RED, "No %s devices found",
+				mp_app_params->devtype_name);
+		return -1;
+	}
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		mp_shared_data->devices_number = nb_devs;
+	} else {
+		if (mp_shared_data->devices_number != nb_devs) {
+			MP_APP_LOG(INFO, COL_RED,
+			"- Number of devices probed by primary process differs with current process config, number of devices = %d, number on primary = %d",
+					nb_devs,
+					mp_shared_data->devices_number);
+			return -1;
+		}
+	}
+
+	for (i = 0; i < nb_devs ; i++) {
+		rte_cryptodev_info_get(valid_devs[i], &info);
+		if (info.feature_flags & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) {
+			mp_app_devs[mp_app_devs_cnt].id = valid_devs[i];
+			mp_app_devs[mp_app_devs_cnt].probed = 1;
+			mp_app_devs[mp_app_devs_cnt++].max_queue_pairs =
+					info.max_nb_queue_pairs;
+
+			/* Last one is as good as first one */
+			mp_app_max_queues = info.max_nb_queue_pairs;
+			if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+				strncpy(mp_shared_data->prim_dev_name[i].name,
+					info.device->name,
+					MP_APP_DEV_NAME_LEN);
+			} else {
+				if (strncmp(
+					mp_shared_data->prim_dev_name[i].name,
+						info.device->name,
+						MP_APP_DEV_NAME_LEN)) {
+					MP_APP_LOG(INFO, COL_RED,
+					"Wrong device: %s, are BDF passed to primary process the same?",
+						info.device->name);
+					return -1;
+				}
+			}
+		}
+	}
+	/* Pick one device to be used for session creation,
+	 * only valid when all devices of the same type.
+	 */
+	mp_app_device_id = mp_app_devs[0].id;
+
+	MP_APP_LOG(INFO, COL_GREEN,
+			"Configure devices according to mask: 0x%lX",
+			mp_app_params->dev_to_configure_mask);
+
+	uint64_t dev_mask_id;
+	int dev_id;
+
+	for (dev_mask_id = 1, dev_id = 0; dev_id <= MP_APP_MAX_DEVS;
+			dev_mask_id <<= 1, dev_id++) {
+		if (dev_mask_id & mp_app_params->dev_to_configure_mask) {
+			if (!mp_app_devs[dev_id].probed)
+				continue;
+
+			/* TODO check if already configured */
+
+			conf.nb_queue_pairs = info.max_nb_queue_pairs;
+			conf.socket_id = SOCKET_ID_ANY;
+			conf.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
+
+			if (rte_cryptodev_configure(mp_app_devs[dev_id].id,
+				&conf) != 0) {
+				RTE_LOG(ERR, USER1,
+					"Error when configuring device number %d",
+						dev_id);
+				return -1;
+			}
+
+			mp_app_devs[dev_id].configured = 1;
+			MP_APP_LOG(INFO, COL_BLUE, "- Configure Device id %d",
+					mp_app_devs[dev_id].id);
+		}
+	}
+	return 0;
+}
+
+static int check_capabilities(int dev_id,
+			const struct mp_crypto_session_vector *vector)
+{
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+
+	cap_idx.type = vector->x_type;
+	if (vector->x_type == RTE_CRYPTO_SYM_XFORM_AEAD)
+		cap_idx.algo.aead = vector->aead_algo;
+
+	/* For now rescricted only to AEAD */
+
+	if (rte_cryptodev_sym_capability_get(dev_id, &cap_idx) == NULL)
+		return -ENOTSUP;
+
+	return 0;
+}
+
+int
+mp_crypto_init_sessions(void)
+{
+	uint64_t session_mask_id;
+	uint64_t session_id;
+	int i;
+	int capabiliy_checked = 1;
+	/* Check if all devices support vector 0 */
+	for (i = 0; i < MP_APP_MAX_VECTORS; i++) {
+		int dev_id = mp_app_devs[i].id;
+		/* TODO use proper vector(s), not hardcoded one */
+		if (dev_id < 0)
+			continue;
+
+		int k = 0;
+
+		while (mp_app_params->enq_param.vector_number[k] >= 0 &&
+				k < MP_APP_MAX_VECTORS) {
+			int vector_number =
+				mp_app_params->enq_param.vector_number[k];
+
+			if (vector_number >= (int)mp_app_numof_ops) {
+				MP_APP_LOG(ERR, COL_RED,
+					"Not recognized test vector %d",
+					vector_number);
+				return -1;
+			}
+			if (check_capabilities(dev_id,
+					&session_vectors[vectors[
+					vector_number].session])) {
+				MP_APP_LOG(ERR, COL_RED,
+					"Algorithm unsupported on dev %d",
+					dev_id);
+				capabiliy_checked = 0;
+			}
+			k++;
+		}
+	}
+	if (capabiliy_checked == 0)
+		return -1;
+
+	for (session_mask_id = 1, session_id = 0;
+			session_id <= mp_app_numof_sessions;
+			session_mask_id <<= 1, session_id++) {
+
+		if (session_mask_id & mp_app_params->session_mask) {
+			struct rte_cryptodev_sym_session *sess =
+				mp_app_create_session(mp_app_device_id,
+				&session_vectors[session_id]);
+			if (sess == NULL) {
+				MP_APP_LOG(ERR, COL_RED,
+					"Error when creating session = %p",
+					sess);
+				return -1;
+			}
+			rte_spinlock_lock(&mp_shared_data->sessions.lock);
+			int clear_session = 1;
+
+			if (mp_shared_data->sessions.sym_sessions[
+					session_id].session
+						== NULL) {
+				mp_shared_data->sessions.sym_sessions[
+					session_id].session
+					= sess;
+				clear_session = 0;
+				/* TODO Remove from spinlock */
+				MP_APP_LOG(INFO, COL_BLUE,
+					"Initialized session = %lu",
+					session_id);
+			} else {
+				/* Actually refcnt should be incremented
+				 * on demand mp_shared_data->sessions.
+				 * sym_sessions [session_id].refcnt++;
+				 */
+			}
+			rte_spinlock_unlock(&mp_shared_data->sessions.lock);
+			if (clear_session)
+				rte_cryptodev_sym_session_free(sess);
+		}
+	}
+	return 0;
+}
+
+int mp_crypto_setup_ops(void)
+{
+	int i;
+	int used_vectors = 0;
+	/* TODO check if device number and qp is correct */
+
+	int selected_vectors[MP_APP_MAX_VECTORS];
+	for (i = 0; i < MP_APP_MAX_VECTORS; i++)
+		selected_vectors[MP_APP_MAX_VECTORS] = -1;
+
+	i = 0;
+	while (mp_app_params->enq_param.vector_number[i] >= 0 &&
+			i < MP_APP_MAX_VECTORS)	{
+		int vector_number = mp_app_params->enq_param.vector_number[i];
+
+		if (mp_app_params->enq_param.vector_number[i] >=
+			(int)mp_app_numof_ops) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Crypto vector %d not defined, skipping",
+				mp_app_params->enq_param.vector_number[i]);
+				i++;
+				continue;
+		}
+		/* Aquire session */
+		int __run = 1;
+		int __continue = 0;
+		int session_id = vectors[vector_number].session;
+
+		while (__run) {
+			int was_session_set = 0;
+
+			rte_spinlock_lock(&mp_shared_data->sessions.lock);
+			if (mp_shared_data->sessions.sym_sessions[
+					session_id].session
+						!= NULL) {
+				mp_shared_data->sessions.sym_sessions[
+					session_id].refcnt++;
+				was_session_set = 1;
+			}
+			rte_spinlock_unlock(&mp_shared_data->sessions.lock);
+			if (was_session_set == 0) {
+				MP_APP_LOG(WARNING, COL_YEL,
+					"Session %d was not yet created, vector %d",
+					session_id, vector_number);
+				char c;
+
+				MP_APP_LOG(INFO, COL_NORM,
+					"Session %d not yet created.\n - Press 'w' to wait until other process will create it \n - Press 'n' to create local session",
+					vectors[session_id].session);
+				int __rte_unused r = scanf("%c", &c);
+
+				if (c == 'n') {
+					struct rte_cryptodev_sym_session *sess =
+						mp_app_create_session(
+							mp_app_device_id,
+						&session_vectors[session_id]);
+					mp_crypto_local_sessions[session_id] =
+						sess;
+					__run = 0;
+				} else if (c == 'w') {
+					int __timeout = 3;
+					int __counter = 1;
+
+					while (__counter <= __timeout) {
+						rte_delay_ms(1000);
+						MP_APP_LOG(INFO, COL_NORM,
+						"Waiting for %d out of %d seconds",
+						__counter++, __timeout);
+					}
+				}
+			} else
+				__run = 0;
+		}
+		if (__continue) {
+			i++;
+			continue;
+		}
+
+		MP_APP_LOG(INFO, COL_BLUE,
+						"Configuring vector %d, using session %d",
+						vector_number, session_id);
+
+		selected_vectors[used_vectors++] = vector_number;
+		i++;
+	}
+
+	if (used_vectors == 0)
+		return 0;
+
+	int curr_vector = 0;
+	/* Create vectors and attach to sessions */
+
+	for (i = 0; i < MP_CRYPTO_QP_DESC_NUM; i++)	{
+		int session_id =
+			vectors[selected_vectors[curr_vector]].session;
+		if (mp_crypto_local_sessions[session_id] != NULL) {
+			mp_crypto_create_op(mp_crypto_ops[i],
+					mp_crypto_mbufs[i],
+					selected_vectors[curr_vector],
+					mp_crypto_local_sessions[session_id]);
+		} else {
+			mp_crypto_create_op(mp_crypto_ops[i],
+					mp_crypto_mbufs[i],
+					selected_vectors[curr_vector],
+					mp_shared_data->sessions.sym_sessions
+					[session_id].session);
+		}
+	}
+	return 0;
+}
+
+int
+mp_crypto_setup_qps(void)
+{
+	int dev_id;
+	int qp_id;
+	int queue_count = 0;
+	int last_qp_on_device = mp_app_max_queues;
+
+	MP_APP_LOG_2(INFO, COL_NORM, "- Configuring queues:");
+	for (dev_id = 0; dev_id < MP_APP_MAX_DEVS; dev_id++) {
+		if (!mp_app_devs[dev_id].probed)
+			continue;
+		for (qp_id = 0; qp_id < mp_app_max_queues; qp_id++) {
+			if (mp_app_devs[dev_id].queue_pair_flag[qp_id]
+					!= QP_TO_CONFIGURE)
+				continue;
+
+			int __run = 1;
+			int __continue = 0;
+
+			while (__run) {
+				/* This could be is_dev_configured */
+				int ret  = rte_cryptodev_get_qp_status(
+					mp_app_devs[dev_id].id, qp_id);
+				if (ret == 1) {
+					mp_app_devs[dev_id].queue_pair_flag[
+						qp_id] = 0;
+					MP_APP_LOG_2(WARNING, COL_YEL,
+						"Queue was already configured by other process, skipping");
+					__run = 0;
+					__continue = 1;
+				} else if (ret < 0) {
+					mp_app_devs[dev_id].queue_pair_flag[
+						qp_id] = 0;
+					MP_APP_LOG_2(ERR, COL_RED,
+						"Error setting queues, was this device configured?");
+					printf(
+						"\n - Press 'w' to wait until other process will configure it");
+					printf("\n - Press 'x' to exit");
+					char c;
+					int __rte_unused r = scanf("%s", &c);
+
+					if (c == 'w') {
+						int __timeout = 3;
+						int __counter = 1;
+
+						while (__timeout <= __counter) {
+							rte_delay_ms(1000);
+							MP_APP_LOG(INFO,
+							COL_NORM,
+							"Waiting for %d out of %d seconds",
+							__counter++, 3);
+						}
+					} else if (c == 'x')
+						return -1;
+				} else if (ret == 0)
+					__run = 0;
+			}
+			if (__continue)
+				continue;
+
+			struct rte_cryptodev_qp_conf qp_conf;
+
+			qp_conf.nb_descriptors = MP_CRYPTO_QP_DESC_NUM;
+			qp_conf.mp_session = NULL;
+			qp_conf.mp_session_private = NULL;
+			if (rte_cryptodev_queue_pair_setup(
+					mp_app_devs[dev_id].id,
+					qp_id, &qp_conf,
+					rte_cryptodev_socket_id(
+					mp_app_devs[dev_id].id))) {
+				RTE_LOG(ERR, USER1,
+					"Error when setting up queue pair %d on dev %d",
+					qp_id, dev_id);
+				return -1;
+			}
+			MP_APP_LOG(INFO, COL_BLUE, "Created qp %d on dev %d",
+					qp_id, mp_app_devs[dev_id].id);
+			mp_app_devs[dev_id].queue_pair_flag[qp_id] = 1;
+			queue_count++;
+		}
+	}
+
+	for (dev_id = 0; dev_id < MP_APP_MAX_DEVS; dev_id++) {
+		if (!mp_app_devs[dev_id].probed)
+			continue;
+		for (qp_id = last_qp_on_device; qp_id < MP_APP_QUEUE_PAIRS_NUM;
+			qp_id++) {
+				if (mp_app_devs[dev_id].queue_pair_flag[qp_id]
+						== QP_TO_CONFIGURE) {
+					MP_APP_LOG(WARNING, COL_YEL,
+					"Cannot create qp %d on dev %d, maximum allowed by this device = %d (%d queue pairs)",
+					qp_id, mp_app_devs[dev_id].id,
+					mp_app_max_queues - 1,
+					mp_app_max_queues);
+				}
+		}
+	}
+
+	MP_APP_LOG(INFO, COL_GREEN, "- Configured %d queues.", queue_count);
+	return 0;
+}
+
+int mp_crypto_setup_mpool(void)
+{
+	int i;
+	char crypto_op_mpool_name[RTE_MEMZONE_NAMESIZE];
+	char mbuf_pool_name[RTE_MEMZONE_NAMESIZE];
+	char session_mpool_name_local[RTE_MEMZONE_NAMESIZE];
+	char session_priv_name_local[RTE_MEMZONE_NAMESIZE];
+
+	/* Op pool */
+	int n = snprintf(crypto_op_mpool_name, sizeof(crypto_op_mpool_name),
+				"%s_%hu", MP_APP_CRYPTO_OP_POOL_NAME,
+					mp_shared_data->proc_counter_total);
+
+	if (n >= (int)sizeof(crypto_op_mpool_name)) {
+		MP_APP_LOG_2(ERR, COL_RED, "Failed to create mpool name");
+		return -1;
+	}
+
+	/* mbuf pool */
+	n = snprintf(mbuf_pool_name, sizeof(mbuf_pool_name),
+				"%s_%hu", MP_APP_MBUFPOOL_NAME,
+				mp_shared_data->proc_counter_total);
+
+	if (n >= (int)sizeof(mbuf_pool_name)) {
+		RTE_LOG(ERR, USER1, "Failed to create mbuf pool name");
+		return -1;
+	}
+
+	/* Local session pool */
+	n = snprintf(session_mpool_name_local,
+			sizeof(session_mpool_name_local),
+			"%s_%hu", MP_APP_SESSION_POOL_NAME_LOC,
+			mp_shared_data->proc_counter_total);
+
+	if (n >= (int)sizeof(session_mpool_name_local)) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to local session mpool name");
+		return -1;
+	}
+
+	/* Local priv session pool */
+	n = snprintf(session_priv_name_local, sizeof(session_priv_name_local),
+				"%s_%hu", MP_APP_PRIV_SESSION_POOL_NAME_LOC,
+				mp_shared_data->proc_counter_total);
+
+	if (n >= (int)sizeof(session_priv_name_local)) {
+		MP_APP_LOG_2(ERR, COL_RED,
+		"Failed to local session private mpool name");
+		return -1;
+	}
+
+	/* Op pool */
+	mp_crypto_op_pool =
+		rte_mempool_lookup(crypto_op_mpool_name);
+
+	if (!mp_crypto_op_pool) {
+		mp_crypto_op_pool = rte_crypto_op_pool_create(
+			crypto_op_mpool_name,
+			RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+			MP_APP_NUM_MBUFS, MP_APP_MBUF_CACHE_SIZE,
+			MP_APP_DEFAULT_NUM_XFORMS *
+			sizeof(struct rte_crypto_sym_xform) +
+			MP_APP_MAXIMUM_IV_LENGTH,
+			rte_socket_id());
+	}
+
+	if (mp_crypto_op_pool == NULL) {
+		MP_APP_LOG_2(ERR, COL_RED, "Error in mempool creation for ops");
+		return -1;
+	}
+
+	/* Set session pools for this process */
+	mp_crypto_session_mempool_local =
+		rte_cryptodev_sym_session_pool_create(
+		session_mpool_name_local, MAX_NUM_OF_SESSIONS,
+		sizeof(struct rte_cryptodev_sym_session), 0, 0,
+		SOCKET_ID_ANY);
+
+	if (!mp_crypto_session_mempool_local) {
+		MP_APP_LOG_2(ERR, COL_RED,
+		"Failed to create local session mpool");
+		return -1;
+	}
+
+	/* Set private session pool for this process */
+	mp_crypto_priv_session_mp_local = rte_mempool_create(
+			session_priv_name_local,
+			MAX_NUM_OF_SESSIONS,
+			rte_cryptodev_sym_get_private_session_size(
+				mp_app_device_id),
+			0, 0, NULL, NULL, NULL,
+			NULL, SOCKET_ID_ANY,
+			0);
+	if (!mp_crypto_priv_session_mp_local) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to create local session priv mpool");
+		return -1;
+	}
+
+	int dev_id = mp_app_devs[0].id;
+	/* All devices use same driver so the same size of private data */
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		/* Set mempools for sessions */
+		mp_crypto_session_mempool =
+			rte_cryptodev_sym_session_pool_create(
+			MP_APP_SESSION_POOL_NAME, MAX_NUM_OF_SESSIONS,
+			sizeof(struct rte_cryptodev_sym_session), 0, 0,
+			SOCKET_ID_ANY);
+
+		if (!mp_crypto_session_mempool) {
+			MP_APP_LOG_2(ERR, COL_RED,
+				"Failed to create session mpool");
+			return -1;
+		}
+
+		mp_crypto_priv_session_mp = rte_mempool_create(
+			MP_APP_PRIV_SESSION_POOL_NAME,
+			MAX_NUM_OF_SESSIONS,
+			rte_cryptodev_sym_get_private_session_size(dev_id),
+			0, 0, NULL, NULL, NULL,
+			NULL, SOCKET_ID_ANY,
+			0);
+		if (!mp_crypto_priv_session_mp) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to create priv mpool");
+			return -1;
+		}
+		/* Set mempools for ops */
+
+	} else {
+		mp_crypto_session_mempool =
+			rte_mempool_lookup(MP_APP_SESSION_POOL_NAME);
+		if (!mp_crypto_session_mempool) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to get sess mpool, was it allocated?");
+			return -1;
+		}
+		mp_crypto_priv_session_mp =
+			rte_mempool_lookup(MP_APP_PRIV_SESSION_POOL_NAME);
+		if (!mp_crypto_session_mempool) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Failed to get priv session mpool, was it allocated?");
+			return -1;
+		}
+	}
+
+	/* Mbuf pool */
+	mp_crypto_mbuf_pool =
+		rte_mempool_lookup(mbuf_pool_name);
+	if (mp_crypto_mbuf_pool == NULL) {
+		mp_crypto_mbuf_pool = rte_pktmbuf_pool_create(
+				mbuf_pool_name,
+				MP_APP_NUM_MBUFS, MP_APP_MBUF_CACHE_SIZE, 0,
+				MP_APP_MBUF_SIZE,
+				rte_socket_id());
+	}
+	if (mp_crypto_mbuf_pool == NULL) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Error in pool creation for mbuf data");
+		return -1;
+	}
+
+	/* Create ops and mbufs */
+	for (i = 0; i < MP_CRYPTO_QP_DESC_NUM; i++)	{
+		mp_crypto_mbufs[i] = rte_pktmbuf_alloc(mp_crypto_mbuf_pool);
+		if (mp_crypto_mbufs[i] == NULL)	{
+			MP_APP_LOG_2(ERR, COL_RED, "Error allocating mbufs");
+			return -1;
+		}
+		memset(rte_pktmbuf_mtod(mp_crypto_mbufs[i], uint8_t *), 0,
+			rte_pktmbuf_data_len(mp_crypto_mbufs[i]));
+	}
+
+	for (i = 0; i < MP_CRYPTO_QP_DESC_NUM; i++) {
+		mp_crypto_ops[i] = rte_crypto_op_alloc(mp_crypto_op_pool,
+			RTE_CRYPTO_OP_TYPE_SYMMETRIC);
+		if (mp_crypto_ops[i] == NULL) {
+			MP_APP_LOG_2(ERR, COL_RED,
+				"Error allocating crypto op");
+			return -1;
+		}
+		mp_crypto_ops[i]->sym->m_src = mp_crypto_mbufs[i];
+	}
+
+	return 0;
+}
+
+static void sigkill_handler(int __rte_unused sig,
+				siginfo_t *siginfo __rte_unused,
+				void *context __rte_unused)
+{
+	mp_crypto_exit_flag = 1;
+	printf("\nInterrupted, finalizing...");
+}
+
+static int
+mp_app_init(int argc, char *argv[])
+{
+	/* init EAL */
+	int ret = rte_eal_init(argc, argv)
+;
+	if (ret < 0)
+		rte_exit(-1, "Invalid EAL arguments!\n");
+
+	argc -= ret;
+	argv += ret;
+
+	struct sigaction sigkill_action;
+
+	memset(&sigkill_action, 0, sizeof(sigkill_action));
+	sigkill_action.sa_sigaction = sigkill_handler;
+	sigkill_action.sa_flags = SA_SIGINFO;
+
+	if (sigaction(SIGINT, &sigkill_action, NULL) < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot init sigation");
+		return -1;
+	}
+
+	if (get_options(argc, argv) != 0) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Get cmdln options returned an error\n");
+		return -1;
+	};
+
+	/* Set driver id for this process */
+	mp_app_driver_id =
+		rte_cryptodev_driver_id_get(mp_app_params->devtype_name);
+	MP_APP_LOG(INFO, COL_BLUE, "- Setting driver %d for this process",
+		mp_app_driver_id);
+
+	/* Register IPC and allocate memzones */
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		MP_APP_LOG_2(INFO, COL_NORM, "- Starting PRIMARY process");
+		if (rte_mp_action_register(MP_APP_IPC_NAME,
+			mp_crypto_primary_handler)) {
+			RTE_LOG(ERR, USER1, "Cannot register IPC callback");
+			return -1;
+		}
+		/* Setup memzone for shared data */
+		mp_app_process_mz = rte_memzone_reserve(MP_APP_PROC_SHARED_NAME,
+				sizeof(struct mp_app_process_data), 0, 0);
+		if (mp_app_process_mz == NULL) {
+			RTE_LOG(ERR, USER1,
+				"mp_app_init: cannot create memzone for process");
+			return -1;
+		}
+		mp_shared_data = mp_app_process_mz->addr;
+		rte_spinlock_init(&mp_shared_data->sessions.lock);
+	} else {
+		MP_APP_LOG_2(INFO, COL_NORM, "- Starting SECONDARY process");
+		if (rte_mp_action_register(MP_APP_IPC_NAME,
+			mp_crypto_secondary_handler)) {
+			RTE_LOG(ERR, USER1, "Cannot register IPC callback");
+			return -1;
+		}
+		/* Setup memzone for shared data */
+		mp_app_process_mz =
+			rte_memzone_lookup(MP_APP_PROC_SHARED_NAME);
+		if (mp_app_process_mz == NULL) {
+			MP_APP_LOG(ERR, COL_RED,
+				"Cannot find memzone by name %s",
+			MP_APP_PROC_SHARED_NAME);
+			return -1;
+		}
+		mp_shared_data = mp_app_process_mz->addr;
+	}
+
+	mp_shared_data->proc_counter++;
+	mp_shared_data->proc_counter_total++;
+	MP_APP_LOG(INFO, COL_GREEN, "Number of processes = %d",
+		mp_shared_data->proc_counter);
+
+	return 0;
+}
+
+void mp_crypto_exit_app(void)
+{
+	const int timeout = 10;
+	int counter = 0;
+	struct rte_mp_msg icp_msg;
+
+	memset(&icp_msg, 0, sizeof(MP_APP_IPC_NAME));
+	mp_crypto_exit_flag = 1;
+	if (mp_shared_data == NULL)
+		return;
+
+/*	rte_mempool_free(mp_crypto_op_pool);
+ *	rte_mempool_free(mp_crypto_mbuf_pool);
+ */
+	rte_mempool_free(mp_crypto_session_mempool_local);
+	rte_mempool_free(mp_crypto_priv_session_mp_local);
+
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		/* Inform of exit intention,
+		 * wait until all processes finish
+		 */
+
+		memcpy(icp_msg.name, MP_APP_IPC_NAME, sizeof(MP_APP_IPC_NAME));
+		memcpy(icp_msg.param, PRIMARY_PROC_EXIT,
+			sizeof(PRIMARY_PROC_EXIT));
+		icp_msg.len_param = sizeof(PRIMARY_PROC_EXIT);
+		icp_msg.num_fds = 0;
+		if (rte_mp_sendmsg(&icp_msg) < 0) {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"Error when sending IPC to secondary processes");
+			return;
+		}
+		while (mp_shared_data->proc_counter > 1 && counter++
+				< timeout) {
+			rte_delay_ms(1000);
+			MP_APP_LOG(INFO, COL_NORM,
+			"Waiting for %d out of %d seconds", counter, timeout);
+		}
+		if (counter < timeout) {
+			MP_APP_LOG_2(INFO, COL_GREEN,
+			"All secondary processes exited normally");
+		} else {
+			MP_APP_LOG_2(ERR, COL_RED,
+			"One or more processes did not exit normally");
+		}
+		rte_mempool_free(mp_crypto_session_mempool);
+		rte_mempool_free(mp_crypto_priv_session_mp);
+
+		mp_shared_data->proc_counter = 0;
+	} else {
+		/* Inform primary of exiting */
+		mp_shared_data->proc_counter--;
+	}
+}
+
+static int check_for_queue(int dev_id, int qp_id)
+{
+	int ret = rte_cryptodev_get_qp_status(dev_id, qp_id);
+
+	if (ret <= 0) {
+		MP_APP_LOG(WARNING, COL_YEL,
+			"Queue %d on dev %d not initialized",
+			qp_id, dev_id);
+		printf(
+		"\n - Press 'w' to wait until other process will initialize it");
+		printf("\n - Press 'x' to exit");
+		char c;
+		int __rte_unused r = scanf("%s", &c);
+
+		if (c == 'w') {
+			int __timeout = 3;
+			int __counter = 1;
+
+			while (__counter <= __timeout) {
+				rte_delay_ms(1000);
+				MP_APP_LOG(INFO, COL_NORM,
+				"Waiting for %d out of %d seconds",
+					__counter++, 3);
+			}
+			return -1;
+		} else if (c == 'x')
+			return -2;
+		else
+			return -2;
+	}
+	return 0;
+}
+
+int mp_crypto_flow(void)
+{
+	int process_enq = 0, process_deq = 0;
+	uint64_t curr_offset_enq = 0;
+	uint64_t curr_offset_deq = 0;
+	uint64_t enqueued = 0;
+	uint64_t dequeued = 0;
+	uint64_t deq_threshold = 0;
+	char c = 0;
+	uint64_t pcks_to_enq = 0, pcks_to_deq = 0;
+
+	int enq_dev_id = mp_app_devs[mp_app_params->enq_param.dev_id].id;
+	int deq_dev_id = mp_app_devs[mp_app_params->deq_param.dev_id].id;
+	int enq_qp_id = mp_app_params->enq_param.qp_id;
+	int deq_qp_id = mp_app_params->deq_param.qp_id;
+	int enq_livesign = 0, deq_livesign = 0;
+	int64_t enq_stall_counter = 0, deq_stall_counter = 0;
+	int livesign_print_idx = 0;
+	int livesign_deq_print_idx = 0;
+
+	if (mp_app_params->enq_param.dev_id >= 0 &&
+			!mp_app_devs[mp_app_params->enq_param.dev_id].probed) {
+		MP_APP_LOG(ERR, COL_RED, "Incorrect enq device provided %d",
+				mp_app_params->enq_param.dev_id);
+	} else if (mp_app_params->enq_param.dev_id >= 0) {
+		MP_APP_LOG(INFO, COL_BLUE,
+			"Start enqueuing packets on dev %d qp %d",
+			mp_app_params->enq_param.dev_id,
+			mp_app_params->enq_param.qp_id);
+		pcks_to_enq = mp_app_params->enq_param.ops_no;
+		process_enq = 1;
+	}
+	if (mp_app_params->deq_param.dev_id >= 0 &&
+			!mp_app_devs[mp_app_params->deq_param.dev_id].probed) {
+		MP_APP_LOG(ERR, COL_RED, "Incorrect deq device provided %d",
+				mp_app_params->deq_param.dev_id);
+	} else if (mp_app_params->deq_param.dev_id >= 0) {
+		MP_APP_LOG(INFO, COL_BLUE,
+			"Start dequeuing packets on dev %d qp %d",
+			mp_app_params->deq_param.dev_id,
+			mp_app_params->deq_param.qp_id);
+		pcks_to_deq = mp_app_params->deq_param.ops_no;
+		process_deq = 1;
+	}
+
+	if (process_enq == 0 && process_deq == 0) {
+		MP_APP_LOG_2(WARNING, COL_YEL, "Nothing to process");
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			while (c != 'k') {
+				printf("\nPress 'k' to exit: ");
+				int __rte_unused r = scanf("%c", &c);
+			}
+		}
+		return 0;
+	}
+
+	/* Check if enq queue was configured */
+	while (process_enq) {
+		int v = check_for_queue(enq_dev_id, enq_qp_id);
+
+		if (v == -1)
+			continue;
+		else if (v == 0)
+			break;
+		else
+			return -1;
+	}
+
+	/* Check if deq queue was configured */
+	while (process_deq) {
+		int v = check_for_queue(deq_dev_id, deq_qp_id);
+
+		if (v == -1)
+			continue;
+		else if (v == 0)
+			break;
+		else
+			return -1;
+	}
+
+	if (process_enq && !process_deq) {
+		MP_APP_LOG_NO_RET(INFO, COL_NORM, "Enqueuing %c",
+			livesign_print_char[livesign_print_idx]);
+	} else if (process_deq && !process_enq) {
+		MP_APP_LOG_NO_RET(INFO, COL_NORM, "Dequeuing %c",
+			livesign_print_char[livesign_deq_print_idx]);
+	} else if (process_enq && process_deq) {
+		MP_APP_LOG_NO_RET(INFO, COL_NORM, "Enqueuing %c Dequeueing %c",
+			livesign_print_char[livesign_print_idx],
+			livesign_print_char[livesign_deq_print_idx]);
+	}
+	while (1) {
+		if (process_enq && !mp_crypto_exit_flag) {
+			if (enqueued < pcks_to_enq || pcks_to_enq == 0) {
+				/* Consider clearing param above */
+				uint64_t __enq;
+				uint64_t to_enq = (MP_CRYPTO_QP_DESC_NUM -
+					curr_offset_enq) > MP_CRYPTO_BURST_NUM ?
+					MP_CRYPTO_BURST_NUM : MP_CRYPTO_QP_DESC_NUM
+					- curr_offset_enq;
+
+				if (pcks_to_enq && to_enq > pcks_to_enq - enqueued)
+					to_enq = pcks_to_enq - enqueued;
+				__enq = rte_cryptodev_enqueue_burst(enq_dev_id,
+					enq_qp_id, &mp_crypto_ops[curr_offset_enq],
+					to_enq);
+				enqueued += __enq;
+				enq_livesign += __enq;
+				curr_offset_enq = enqueued % MP_CRYPTO_QP_DESC_NUM;
+				if (enq_livesign > mp_app_params->enq_param.checkpoint) {
+					if (process_enq && !process_deq) {
+						MP_APP_LOG_NO_RET(INFO, COL_NORM,
+							"Enqueuing %c",
+							livesign_print_char[livesign_print_idx]);
+					}
+					if (process_enq && process_deq) {
+						MP_APP_LOG_NO_RET(INFO, COL_NORM,
+						"Enqueuing %c Dequeueing %c",
+						livesign_print_char[livesign_print_idx],
+						livesign_print_char[livesign_deq_print_idx]);
+					}
+					livesign_print_idx++;
+					livesign_print_idx %= 4;
+					enq_livesign = 0;
+				}
+				if (__enq == 0)
+					enq_stall_counter++;
+				else
+					enq_stall_counter = 0;
+			}
+		}
+
+		if (process_deq) {
+			if (dequeued < pcks_to_deq || pcks_to_deq == 0) {
+				uint64_t __deq;
+				uint64_t to_deq = (MP_CRYPTO_QP_DESC_NUM -
+					curr_offset_deq)
+					> MP_CRYPTO_BURST_NUM ?	MP_CRYPTO_BURST_NUM :
+					MP_CRYPTO_QP_DESC_NUM - curr_offset_deq;
+
+				if (pcks_to_deq && to_deq > pcks_to_deq - dequeued)
+					to_deq = pcks_to_deq - dequeued;
+				__deq = rte_cryptodev_dequeue_burst(deq_dev_id,
+					deq_qp_id, &mp_crypto_ops_ret[curr_offset_deq],
+					to_deq);
+				dequeued += __deq;
+				deq_livesign += __deq;
+				curr_offset_deq = dequeued % MP_CRYPTO_QP_DESC_NUM;
+				if (deq_livesign > mp_app_params->deq_param.checkpoint) {
+					if (process_deq && !process_enq) {
+						MP_APP_LOG_NO_RET(INFO, COL_NORM,
+						"Dequeueing %c",
+						livesign_print_char[livesign_deq_print_idx]);
+					}
+					if (process_enq && process_deq) {
+						MP_APP_LOG_NO_RET(INFO, COL_NORM,
+						"Enqueuing %c Dequeueing %c",
+						livesign_print_char[livesign_print_idx],
+						livesign_print_char[livesign_deq_print_idx]);
+					}
+					livesign_deq_print_idx++;
+					livesign_deq_print_idx %= 4;
+					deq_livesign = 0;
+				}
+				if (__deq == 0)
+					deq_stall_counter++;
+				else
+					deq_stall_counter = 0;
+				if (mp_crypto_exit_flag) {
+					deq_threshold += __deq;
+					if (deq_threshold > 100000)
+						break;
+					if (deq_stall_counter > 100000)
+						break;
+				}
+			}
+		}
+
+		if (((dequeued == pcks_to_deq && process_deq)) &&
+			 ((enqueued == pcks_to_enq && process_enq))) {
+			MP_APP_LOG(INFO, COL_GREEN,
+					"\nEnqueued %lu, dequeued %lu packets",
+					enqueued, dequeued);
+			break;
+		} else if (dequeued == pcks_to_deq && process_deq &&
+				!process_enq && pcks_to_deq)  {
+			MP_APP_LOG(INFO, COL_GREEN, "\nDequeued %lu packets",
+				dequeued);
+			break;
+		} else if (enqueued == pcks_to_enq && process_enq &&
+				!process_deq && process_enq)  {
+			MP_APP_LOG(INFO, COL_GREEN, "\nEnqueued %lu packets",
+				enqueued);
+			break;
+		}
+		if (mp_crypto_exit_flag && !process_deq)
+			break;
+	}
+
+	/* Verify if all packets are correct */
+	if (process_deq) {
+		uint64_t last_packet = pcks_to_deq > MP_CRYPTO_QP_DESC_NUM ?
+			MP_CRYPTO_QP_DESC_NUM : pcks_to_deq;
+		if (pcks_to_deq == 0)
+			last_packet = MP_CRYPTO_QP_DESC_NUM;
+		if (last_packet >= dequeued)
+			last_packet = dequeued;
+		uint64_t k;
+		int err = 0;
+
+		for (k = 0; k < last_packet; k++) {
+			if (mp_crypto_ops_ret[k]->status !=
+				RTE_CRYPTO_OP_STATUS_SUCCESS) {
+				MP_APP_LOG(ERR, COL_RED,
+					"error when checking status of %lu packet out of last %lu packets",
+					k, last_packet);
+					err = 1;
+					break;
+			}
+		}
+		if (err == 0) {
+			MP_APP_LOG(INFO, COL_GREEN,
+				"\nAll %lu last packets verified correctly",
+				last_packet);
+		}
+	}
+
+	if (mp_app_params->print_stats) {
+		struct rte_cryptodev_stats stats;
+
+		if (enq_qp_id >= 0) {
+			rte_cryptodev_stats_get(enq_dev_id, &stats);
+			MP_APP_LOG(INFO, COL_BLUE,
+				"STATS: Enqueued on dev %d, qp %d = %lu",
+				enq_dev_id, enq_qp_id, stats.enqueued_count);
+			MP_APP_LOG(INFO, COL_BLUE,
+				"STATS: Enqueue err count on dev %d, qp %d = %lu",
+				enq_dev_id, enq_qp_id,
+				stats.enqueue_err_count);
+		}
+		if (deq_qp_id >= 0) {
+			rte_cryptodev_stats_get(deq_dev_id, &stats);
+			MP_APP_LOG(INFO, COL_BLUE,
+				"STATS: Dequeued on dev %d, qp %d = %lu",
+				deq_dev_id, deq_qp_id, stats.dequeued_count);
+			MP_APP_LOG(INFO, COL_BLUE,
+				"STATS: Dequeue err count on dev %d, qp %d = %lu",
+				deq_dev_id, deq_qp_id, stats.dequeue_err_count);
+		}
+
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			while (c != 'k') {
+				printf("\nPress 'k' to exit: ");
+				int __rte_unused r = scanf("%c", &c);
+			}
+		}
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	int ret = 0;
+
+	if (mp_app_init(argc, argv) < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Error when initializing");
+		goto err;
+	};
+
+	if (mp_crypto_init_devs() < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Devices cannot be initialized");
+		goto err;
+	};
+
+	ret = mp_crypto_setup_mpool();
+	if (ret < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot create mempools");
+		goto err;
+	}
+
+	if (mp_crypto_setup_qps() < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Setup qps returned an error");
+		goto err;
+	};
+
+	ret = mp_crypto_init_sessions();
+	if (ret < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot initialize sessions");
+		goto err;
+	}
+
+	ret = mp_crypto_setup_ops();
+	if (ret < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot setup ops");
+		goto err;
+	}
+
+	ret = mp_crypto_flow();
+	if (ret < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Cannot enq/deq");
+		goto err;
+	}
+
+
+	mp_crypto_exit_app();
+	return 0;
+err:
+	mp_crypto_exit_app();
+
+	return 1;
+}
diff --git a/examples/mp_crypto/meson.build b/examples/mp_crypto/meson.build
new file mode 100644
index 0000000..08976ab
--- /dev/null
+++ b/examples/mp_crypto/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2020 Intel Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+if not dpdk_conf.has('RTE_LIBRTE_CRYPTODEV')
+        build = false
+        subdir_done()
+endif
+
+allow_experimental_apis = true
+deps += ['cryptodev']
+sources = files(
+	'main.c', 'mp_crypto.c', 'mp_crypto_parser.c', 'mp_crypto_vectors.c', 'mp_crypto_ipc.c'
+)
diff --git a/examples/mp_crypto/mp_crypto.c b/examples/mp_crypto/mp_crypto.c
new file mode 100644
index 0000000..d041d35
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto.c
@@ -0,0 +1,139 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#include "mp_crypto_vectors.h"
+#include "mp_crypto.h"
+#include "mp_crypto_parser.h"
+
+int			mp_app_driver_id;
+/* Global driver id, one per mp_app */
+int			mp_app_device_id;
+/* For now we use only one device type, so for session
+ * init only one need to be provided
+ */
+struct mp_app_dev	mp_app_devs[MP_APP_MAX_DEVS];
+/* Global devices list */
+uint16_t		mp_app_devs_cnt;
+/* Global device counter */
+uint8_t			mp_app_max_queues;
+/* Per process queue counter */
+const struct rte_memzone *mp_app_process_mz;
+struct mp_app_process_data *mp_shared_data;
+/* Data shared across processes
+ * memzone name = MP_PROC_SHARED_MZ
+ */
+
+int mp_crypto_exit_flag;
+/* Global exit flag */
+
+struct rte_mempool *mp_crypto_session_mempool;
+/* Global crypto mempool used by all processes */
+struct rte_mempool *mp_crypto_session_mempool_local;
+/* Local crypto mempool used by this process */
+struct rte_mempool *mp_crypto_priv_session_mp;
+/* Global crypto private session mempool used by all processes */
+struct rte_mempool *mp_crypto_priv_session_mp_local;
+/* Local crypto private session mempool used by this process */
+struct rte_mempool *mp_crypto_op_pool;
+/* Per process op pool */
+struct rte_mempool *mp_crypto_mbuf_pool;
+/* Per process mbuf pool */
+
+struct rte_cryptodev_sym_session *mp_crypto_local_sessions[MAX_NUM_OF_SESSIONS];
+/* Array of private sessions */
+
+struct rte_crypto_op *mp_crypto_ops[MP_CRYPTO_OPS_NUM];
+/* Per process set of rte crypto ops */
+struct rte_crypto_op *mp_crypto_ops_ret[MP_CRYPTO_OPS_NUM];
+/* Per process set of return rte crypto ops */
+struct rte_mbuf *mp_crypto_mbufs[MP_CRYPTO_OPS_NUM];
+/* Per process set of rte mbufs */
+
+/* Function for creating sessions */
+struct rte_cryptodev_sym_session *mp_app_create_session
+	(int dev_id, const struct mp_crypto_session_vector *vector)
+{
+	if (vector->x_type == RTE_CRYPTO_SYM_XFORM_AEAD)
+		return mp_app_create_aead_session(dev_id, vector);
+	MP_APP_LOG_2(ERR, COL_RED, "Invalid xform type");
+	return NULL;
+}
+
+/* Create AEAD session */
+struct rte_cryptodev_sym_session*
+mp_app_create_aead_session(int dev_id,
+		const struct mp_crypto_session_vector *vector)
+{
+	struct rte_cryptodev_sym_session *session;
+	struct rte_crypto_sym_xform xform;
+
+	xform.next = NULL;
+	xform.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	xform.aead.key.length = vector->crypto_key.len;
+	xform.aead.key.data = vector->crypto_key.data;
+	xform.aead.algo = vector->aead_algo;
+	xform.aead.digest_length = vector->digest_len;
+	xform.aead.iv.length = vector->iv_len;
+	xform.aead.iv.offset = IV_OFFSET;
+	xform.aead.aad_length = vector->aad_len;
+	xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
+
+	session = rte_cryptodev_sym_session_create(mp_crypto_session_mempool);
+	if (session == NULL) {
+		MP_APP_LOG_2(ERR, COL_RED, "Failed to create session");
+		return NULL;
+	}
+	int status = rte_cryptodev_sym_session_init(dev_id, session,
+			&xform,	mp_crypto_priv_session_mp);
+	if (status < 0) {
+		MP_APP_LOG_2(ERR, COL_RED, "Failed to init session");
+		return NULL;
+	}
+
+	return session;
+}
+
+int
+mp_crypto_create_op(struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+			uint16_t vector_number,
+			struct rte_cryptodev_sym_session *sess)
+{
+	uint8_t *plaintext;
+	uint32_t aad_pad_len =
+		RTE_ALIGN_CEIL(session_vectors[vectors[vector_number].
+				session].aad_len, 16);
+
+	memset(rte_pktmbuf_mtod(mbuf, uint8_t *), 0,
+			rte_pktmbuf_tailroom(mbuf));
+	struct rte_crypto_sym_op *sym_op = op->sym;
+
+	sym_op->aead.aad.data = (uint8_t *)rte_pktmbuf_append(mbuf,
+			aad_pad_len);
+	sym_op->aead.aad.phys_addr =
+			rte_pktmbuf_iova(mbuf);
+	memcpy(sym_op->aead.aad.data, vectors[vector_number].aad.data,
+		session_vectors[vectors[vector_number].session].aad_len);
+	uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op,
+			uint8_t *, IV_OFFSET);
+	rte_memcpy(iv_ptr, vectors[vector_number].iv,
+		session_vectors[vectors[vector_number].session].iv_len);
+
+	plaintext = (uint8_t *)rte_pktmbuf_append(mbuf,
+			vectors[vector_number].plaintext.len);
+	rte_memcpy(plaintext, vectors[vector_number].plaintext.data,
+			vectors[vector_number].plaintext.len);
+
+	sym_op->aead.digest.phys_addr =
+		rte_pktmbuf_iova_offset(mbuf,
+			vectors[vector_number].plaintext.len);
+
+	sym_op->aead.digest.data = (uint8_t *)rte_pktmbuf_append(
+			mbuf, vectors[vector_number].digest.len);
+
+	sym_op->aead.data.length = vectors[vector_number].plaintext.len;
+	sym_op->aead.data.offset = 0;
+
+	if (rte_crypto_op_attach_sym_session(op, sess))
+		return -1;
+	return 0;
+}
diff --git a/examples/mp_crypto/mp_crypto.h b/examples/mp_crypto/mp_crypto.h
new file mode 100644
index 0000000..e708ee3
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto.h
@@ -0,0 +1,224 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#ifndef _MP_CRYPTO_SAMPLE_APP_
+#define _MP_CRYPTO_SAMPLE_APP_
+
+#include <stdint.h>
+#include <rte_hexdump.h>
+#include "mp_crypto_vectors.h"
+
+/* Intel QuickAssist Technology Symmetric service PMD name */
+#define CRYPTODEV_NAME_QAT_SYM_PMD	"crypto_qat"
+/* Maximum number of devices to configure with this app */
+#define MP_APP_MAX_DEVS			64
+/* Maximum number of queue pairs per device */
+#define MP_APP_QUEUE_PAIRS_NUM		8
+
+#define MP_APP_PROC_SHARED_NAME		"MP_PROC_SHARED_MZ"
+/* Memzone name for shared data across processes */
+#define MP_APP_IPC_NAME			"MP_APP_IPC_NAME"
+
+/* Session pool information */
+#define MP_APP_SESSION_POOL_NAME	"MP_APP_SESSION_POOL_NAME"
+#define MP_APP_PRIV_SESSION_POOL_NAME	"MP_APP_PRIV_SESSPOL_NAME"
+
+#define MP_APP_SESSION_POOL_NAME_LOC		"MP_APP_SESSP_NAME_LOC"
+#define MP_APP_PRIV_SESSION_POOL_NAME_LOC	"MP_APP_PRIV_SPOL_NLOC"
+
+#define MAX_NUM_OF_SESSIONS		(16)
+
+/* Crypto op information */
+#define MP_APP_CRYPTO_OP_POOL_NAME	"MP_APP_OP_NAME"
+/* Mbuf information */
+#define MP_APP_MBUFPOOL_NAME		"MP_APP_MBUF_NAME"
+
+extern int mp_crypto_exit_flag;
+/* Global exit flag */
+
+/*
+ * IPC COMMANDS
+ */
+#define PRIMARY_PROC_EXIT		"PRIMARY_EXIT"
+#define SECONDARY_PROC_EXIT		"SECONDARY_EXIT"
+
+#define MP_APP_DEV_NAME_LEN	64
+/* Max name length */
+
+/* Op pool constants */
+#define MP_APP_NUM_MBUFS					(4096)
+/* Same number as default/max ops */
+#define MP_APP_MBUF_CACHE_SIZE				(256)
+#define MP_APP_DEFAULT_NUM_XFORMS			(2)
+#define MP_APP_MAXIMUM_IV_LENGTH			(16)
+/* Mbuf constants */
+#define MP_APP_MBUF_SIZE			(sizeof(struct rte_mbuf) + \
+		RTE_PKTMBUF_HEADROOM + MBUF_DATAPAYLOAD_SIZE)
+/* qps constants */
+#define MP_CRYPTO_QP_DESC_NUM		(4096)
+#define NP_CRYPTO_OPS_TO_ENQ		(160000)
+#define NP_CRYPTO_OPS_TO_DEQ		(160000)
+/* Enqueue constants */
+#define MP_CRYPTO_BURST_NUM		(64)
+#define MP_CRYPTO_OPS_NUM		(MP_APP_NUM_MBUFS)
+/* Device information */
+#define MP_CRYPTO_MAX_DEVS		(64)
+
+extern struct rte_crypto_op *mp_crypto_ops[];
+/* Per process set of rte crypto ops */
+extern struct rte_crypto_op *mp_crypto_ops_ret[];
+/* Per process set of return rte crypto ops */
+extern struct rte_mbuf *mp_crypto_mbufs[];
+/* Per process set of rte mbufs */
+
+/* Name of the device */
+struct mp_app_dev_name {
+	char name[MP_APP_DEV_NAME_LEN];
+};
+
+extern struct rte_cryptodev_sym_session *mp_crypto_local_sessions[];
+/* Array of private sessions */
+
+/* Symmetric session + ref count*/
+struct mp_app_shared_sym_session {
+	struct rte_cryptodev_sym_session *session;
+	/* Pointer to symmetric session */
+	int refcnt;
+	/* Reference count, process that created this session
+	 * does not increment this value
+	 */
+};
+
+/* Data for session array to be shared */
+struct mp_app_session_array {
+	struct mp_app_shared_sym_session sym_sessions[MAX_NUM_OF_SESSIONS];
+	/* Array of pointers to sessions */
+	int sym_session_counter;
+	/* Counter of allocated sessions */
+	rte_spinlock_t lock;
+	/* Spinlock guarding this array */
+};
+
+/* Data to be shared across processes */
+struct mp_app_process_data {
+	uint16_t proc_counter;
+	/* Counter of processes */
+	uint16_t proc_counter_total;
+	/* Number of processes that joined, not decremented
+	 * can be used for naming in particular processes
+	 */
+	uint16_t devices_number;
+	/* Number of devices probed by primary process */
+	struct mp_app_dev_name prim_dev_name[MP_APP_MAX_DEVS];
+	/* Names of devices probed by primary process */
+	struct mp_app_session_array sessions;
+	/* Array of sessions to be visible by all processes */
+};
+
+extern const struct rte_memzone *mp_app_process_mz;
+extern struct mp_app_process_data *mp_shared_data;
+/* Data shared across processes
+ * memzone name = MP_PROC_SHARED_MZ
+ */
+
+extern struct rte_mempool *mp_crypto_session_mempool;
+/* Global crypto session mempool used by all processes */
+extern struct rte_mempool *mp_crypto_session_mempool_local;
+/* Local crypto mempool used by this process */
+extern struct rte_mempool *mp_crypto_priv_session_mp;
+/* Global crypto private session mempool used by all processes */
+extern struct rte_mempool *mp_crypto_priv_session_mp_local;
+/* Local crypto private session mempool used by this process */
+extern struct rte_mempool *mp_crypto_op_pool;
+/* Per process op pool */
+extern struct rte_mempool *mp_crypto_mbuf_pool;
+/* Per process mbuf pool */
+
+struct mp_app_dev {
+	int8_t id;
+	/* Cryptodev id of this dev */
+	int queue_pair_flag[MP_APP_QUEUE_PAIRS_NUM];
+	/* 1 means qp was configured for this device,
+	 * 0 not configured by this process, but still
+	 * could be initialized by another
+	 * -2 means this qp is to be configured
+	 */
+	uint16_t max_queue_pairs;
+	/* Per device info */
+	uint8_t probed;
+	/* If device was probed by EAL */
+	uint8_t configured;
+	/* Was this device configured */
+	const struct rte_memzone *shared_data;
+	/* This data is shared across processes
+	 * memzone name = MZ_DEV_SHARED_DATA_DEV_[ID]
+	 */
+};
+
+extern int			mp_app_driver_id;
+/* Global driver id, one per mp_app */
+extern int			mp_app_device_id;
+/* For now we use only one device type, so for session
+ * init only one need to be provided
+ */
+extern struct mp_app_dev	mp_app_devs[];
+/* Global devices list */
+extern uint16_t			mp_app_devs_cnt;
+/* Global device counter */
+extern uint8_t			mp_app_max_queues;
+/* Per process queue counter */
+
+void mp_crypto_exit_app(void);
+/* Exit function for both primary and secondary */
+
+int mp_crypto_setup_mpool(void);
+/* Function to set or lookup for mempools */
+
+int mp_crypto_flow(void);
+/* Flow function for enqueue dequeue */
+
+/*
+ * Primary process IPC handler
+ */
+int
+mp_crypto_primary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer);
+int
+mp_crypto_secondary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer);
+
+int mp_crypto_setup_qps(void);
+/* Function to setup queues according to input string */
+
+int mp_crypto_init_sessions(void);
+/* Function to setup session according to mask */
+
+int mp_crypto_init_devs(void);
+/* Function to setup devices according to mask */
+
+int mp_crypto_setup_ops(void);
+/* Function to setup opse according to input string enq=[] */
+
+/* Create and init symmetric session */
+struct rte_cryptodev_sym_session *mp_app_create_session
+		(int dev_id, const struct mp_crypto_session_vector *vector);
+
+/* Create AEAD session */
+struct rte_cryptodev_sym_session*
+		mp_app_create_aead_session(int dev_id,
+		const struct mp_crypto_session_vector *vector);
+
+/* Create op */
+int
+mp_crypto_create_op(struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+					uint16_t vector_number,
+					struct rte_cryptodev_sym_session *sess);
+
+#define IV_OFFSET			(sizeof(struct rte_crypto_op) + \
+		sizeof(struct rte_crypto_sym_op) + DEFAULT_NUM_XFORMS * \
+		sizeof(struct rte_crypto_sym_xform))
+
+#define MBUF_DATAPAYLOAD_SIZE		(2048)
+#define DEFAULT_NUM_XFORMS			(2)
+
+#endif
diff --git a/examples/mp_crypto/mp_crypto_ipc.c b/examples/mp_crypto/mp_crypto_ipc.c
new file mode 100644
index 0000000..9d5a8cb
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_ipc.c
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#include "mp_crypto.h"
+
+/*
+ * Primary process IPC handler
+ */
+int
+mp_crypto_primary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer)
+{
+	(void)peer;
+	if (!memcmp(SECONDARY_PROC_EXIT, (const char *)mp_msg->param,
+		sizeof(SECONDARY_PROC_EXIT))) {
+		RTE_LOG(ERR, USER1, "One of secondary processes exiting...");
+	}
+	return 0;
+}
+
+int
+mp_crypto_secondary_handler(const struct rte_mp_msg *mp_msg,
+		  const void *peer)
+{
+	(void)peer;
+	if (!memcmp(PRIMARY_PROC_EXIT, (const char *)mp_msg->param,
+		sizeof(PRIMARY_PROC_EXIT)))	{
+		RTE_LOG(ERR, USER1, "Primary process exiting...");
+		mp_crypto_exit_flag = 1;
+	}
+	return 0;
+}
diff --git a/examples/mp_crypto/mp_crypto_parser.c b/examples/mp_crypto/mp_crypto_parser.c
new file mode 100644
index 0000000..165f345
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_parser.c
@@ -0,0 +1,511 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include <getopt.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <errno.h>
+
+#include <rte_string_fns.h>
+#include <rte_comp.h>
+
+#include "mp_crypto_parser.h"
+#include "mp_crypto.h"
+
+struct mp_crypto_app_parameters *mp_app_params;
+
+static void
+usage(char *progname)
+{
+	/* TODO, find better way of formatting columns... */
+	printf("%s [EAL options] -- [options]"
+	"\noptions:"
+	"\n  --devtype [device name]: \t\t\tdevice name, the same name need to be used"
+	" across all processes. \n\t\t\t\t\t\t--Example: --devtype=crypto_qat"
+	"\n  --config-dev [dev_id,]: \t\t\tid of device that should be"
+	" configured by this process. Note that order of ids depends on the"
+	" Cryptodev\n\t\t\t\t\t\tglobal array placement so BDF of smaller numbers will come"
+	" first. \n\t\t\t\t\t\t--Example: -w 03:01.2 -w 03:01.1 -w 03:01.3 --config-dev 0,2"
+	" will configure devices 03:01.1 and 03:01.3."
+	"\n  --qp-config=[dev_id]:[qp_id,];...: \t\tqueue_pairs qp_id's to be configured dev_id's"
+	"\n\t\t\t\t\t\t--Example: --qp-config=0:0,1;1:1;0:1; - will configure qp's 0,1 on device 0"
+	"' 1 on device 1, 0,1 on device 2.'"
+	"\n  --session-mask=[mask]\t\t\t\tsession to be shared for all processes, session list is in"
+	" mp_crypto_vectors.c file.\n\t\t\t\t\t\tIf session mask will not be set it still can be configured"
+	" interactively by user for certain process and the used by this process only"
+	"\n\t\t\t\t\t\t--Example --sesion-mask=0x3 will configure session 0 and 1."
+	"\n  --enq=[dev_id]:[qp_id]:[ops]:[vector_id]:\tEnqueue operation for this process"
+	"\n\t\t\t\t\t\t- dev_id: device selected the same way as in --config-dev option"
+	"\n\t\t\t\t\t\t- qp_id: queue pair to bu used for enqueue operation"
+	"\n\t\t\t\t\t\t- ops: 0 means it will run in infinite loop (ctrl-c will inform other processes),"
+	"other than that any positive number"
+	"\n\t\t\t\t\t\t- vector_id: vector id to be used, vector array can be found"
+	" in mp_crypto_vectors.c file. "
+	"\n\t\t\t\t\t\t- Only one can be specified by process"
+	"\n  --deq=[dev_id]:[qp_id]:[ops]:[vector_id]:\tDequeue operation for this process"
+	"\n\t\t\t\t\t\t- dev_id: device selected the same way as in --config-dev option"
+	"\n\t\t\t\t\t\t- qp_id: queue pair to bu used for dequeue operation"
+	"\n\t\t\t\t\t\t- ops: 0 means it will run in infinite loop (ctrl-c will inform other processes),"
+	"other than that any positive number"
+	"\n\t\t\t\t\t\t- vector_id: vector id to be used, vector array can be found"
+	" in mp_crypto_vectors.c file. "
+	"\n\t\t\t\t\t\t- Only one can be specified by process"
+	"\n  --print-stats: \t\t\t\tPrint stats at then end of program."
+	"\n",
+	progname);
+}
+
+static struct option lgopts[] = {
+	{ MP_DEV_CONFIGURE, required_argument, 0, 0 },
+	{ MP_QP_CONFIGURE, required_argument, 0, 0 },
+	{ MP_ENQ, required_argument, 0, 0 },
+	{ MP_DEQ, required_argument, 0, 0 },
+	{ MP_SESSION_MASK, required_argument, 0, 0 },
+	{ MP_PRINT_STATS, 0, 0, 0 },
+	{ MP_DEVTYPE_NAME, required_argument, 0, 0 },
+	{ NULL, 0, 0, 0 }
+};
+
+static void dump_test_data_options(struct mp_crypto_app_parameters
+	*test_data __rte_unused) {
+}
+
+int16_t
+get_options(int argc, char *argv[]) {
+	mp_app_params = rte_zmalloc_socket(NULL,
+					sizeof(struct mp_crypto_app_parameters),
+					0, rte_socket_id());
+
+	if (mp_app_params == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Failed to allocate for test data\n");
+		return -1;
+	}
+
+	options_default(mp_app_params);
+
+	if (options_parse(mp_app_params, argc, argv) != 0) {
+		MP_APP_LOG_2(ERR, COL_RED,
+			"Parsing one or more user options failed");
+		return -1;
+	}
+
+	if (options_check(mp_app_params) != 0) {
+		RTE_LOG(ERR, USER1,
+			"Inconsistent user options.\n");
+		dump_test_data_options(mp_app_params);
+		return -1;
+	}
+
+	dump_test_data_options(mp_app_params);
+	return 0;
+}
+
+static int
+parse_config_dev(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	char *end = NULL;
+	const char *start = arg;
+	uint64_t num;
+	char str[32];
+
+	while (1) {
+		memset(str, 0, sizeof(str));
+		end = strchr(start, ',');
+		if (end) {
+			memcpy(str, start, end - start);
+			errno = 0;
+			num = strtoull(str, NULL, 10);
+			if (errno) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Invalid device provided '%s'", str);
+				return -1;
+			}
+			if (num >= MP_CRYPTO_MAX_DEVS) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Device number not supported %lu", num);
+				return -1;
+			}
+			/* Sanity check, unfortunately c standard does not
+			 * force errno to be set when no conversion
+			 * can by performed (except for ERANGE)
+			 */
+			if (num == 0) {
+				if (start[0] != '0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+				if (start[1] != ',') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+			}
+			mp_params->dev_to_configure_mask |= 1LU << (num);
+			start = end + 1;
+			if (*start == 0)
+				break;
+		} else {
+			end = strchr(start, '\0');
+			memcpy(str, start, end - start);
+			errno = 0;
+			num = strtoull(str, NULL, 10);
+			if (errno) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Invalid device provided '%s'", str);
+				return -1;
+			}
+			if (num >= 64) {
+				MP_APP_LOG(ERR, COL_RED,
+				"Device number not supported %lu", num);
+				return -1;
+			}
+			/* Sanity check, unfortunately c standard does not force
+			 * errno to be set when no conversion can by performed
+			 * (except for ERANGE)
+			 */
+			if (num == 0) {
+				if (start[0] != '0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+				if (start[1] != '\0') {
+					MP_APP_LOG(ERR, COL_RED,
+					"Invalid device provided '%s'", str);
+					return -1;
+				}
+			}
+			mp_params->dev_to_configure_mask |= 1LU << (num);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* Veeeery simple parser */
+static int mp_parse_qps(const char *arg)
+{
+	char str[64] = { };
+	int dev_id = -1;
+	const char *start = arg;
+	const char *end;
+	int finish = 0;
+
+	while (1) {
+		end = strchr(start, ':');
+		if (end == NULL)
+			return 0;
+		memcpy(str, start, end - start);
+		dev_id = strtol(str, NULL, 10);
+		start = end + 1;
+		if (*start == '\0') {
+			MP_APP_LOG_2(ERR, COL_RED,
+				"Parsing queue pairs: error parsing");
+			return -1;
+		}
+		const char *curr = start;
+
+		while (1) {
+			memset(str, 0, sizeof(str));
+			if (*curr == ',' || *curr == ';' || *curr == '\0') {
+				memcpy(str, start, curr - start);
+				int qp_id = strtol(str, NULL, 10);
+
+				if (qp_id > (MP_APP_QUEUE_PAIRS_NUM - 1)) {
+					MP_APP_LOG(WARNING, COL_YEL, "Cannot create qp: %d, maximum qp number allowed %d (%d queues)",
+					qp_id, MP_APP_QUEUE_PAIRS_NUM - 1,
+					MP_APP_QUEUE_PAIRS_NUM);
+				}
+
+				mp_app_devs[dev_id].queue_pair_flag[qp_id] =
+						QP_TO_CONFIGURE;
+			}
+			if (*curr == ',') {
+				start = curr + 1;
+				curr++;
+				continue;
+			} else if (*curr == ';') {
+				start = curr + 1;
+				break;
+			} else if (*curr == '\0') {
+				finish = 1;
+				break;
+			}
+			curr++;
+		}
+		if (finish)
+			break;
+	}
+
+	return 0;
+}
+
+static int
+parse_qp_config(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	strncpy(mp_params->qp_config, arg, MP_APP_QP_PARAM_LEN);
+	if (mp_parse_qps(arg)) {
+		MP_APP_LOG_2(ERR, COL_RED, "- Parsing error, qpairs string");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+parse_enq(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	char str[64] = { };
+	const char *start = arg;
+	/* dev id */
+	char *end = strchr(start, ':');
+	int i = 0;
+
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.dev_id = strtol(str, NULL, 10);
+	/* qp id */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.qp_id = strtol(str, NULL, 10);
+	/* ops no */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->enq_param.ops_no = strtol(str, NULL, 10);
+	/* vector ids */
+	start = end + 1;
+	while ((end = strchr(start, ',')) != NULL) {
+		memset(str, 0, sizeof(str));
+		memcpy(str, start, end - start);
+		mp_params->enq_param.vector_number[i] = strtoul(str, NULL, 0);
+		start = end + 1;
+		i++;
+	}
+	if (i == 0)
+		goto err;
+
+	MP_APP_LOG(INFO, COL_BLUE, "Run enqueue on device %d",
+			mp_params->enq_param.dev_id);
+	MP_APP_LOG(INFO, COL_BLUE, "Run enqueue on qp %d",
+			mp_params->enq_param.qp_id);
+	i = 0;
+	while (mp_params->enq_param.vector_number[i] > 0 &&
+			i < MP_APP_MAX_VECTORS)	{
+		MP_APP_LOG(INFO, COL_BLUE, "Run enqueue vector %d",
+			mp_params->enq_param.vector_number[i]);
+		i++;
+	}
+
+	mp_params->enq_param.checkpoint = 1000000;
+
+	return 0;
+err:
+	MP_APP_LOG_2(ERR, COL_RED, "Error parsing enq");
+	return -1;
+}
+
+static int
+parse_deq(struct mp_crypto_app_parameters *mp_params, const char *arg)
+{
+	char str[64] = { };
+	const char *start = arg;
+	/* Dev id */
+	char *end = strchr(start, ':');
+	int i = 0;
+
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.dev_id = strtol(str, NULL, 10);
+	/* qp id */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.qp_id = strtol(str, NULL, 10);
+	/* ops no */
+	memset(str, 0, sizeof(str));
+	start = end + 1;
+	end = strchr(start, ':');
+	if (end == NULL)
+		goto err;
+	memcpy(str, start, end - start);
+	mp_params->deq_param.ops_no = strtol(str, NULL, 10);
+
+	/* vector no */
+	start = end + 1;
+	while ((end = strchr(start, ',')) != NULL) {
+		memset(str, 0, sizeof(str));
+		memcpy(str, start, end - start);
+		mp_params->deq_param.vector_number[i] = strtoul(str, NULL, 0);
+		start = end + 1;
+		i++;
+	}
+	if (i == 0)
+		goto err;
+
+	MP_APP_LOG(INFO, COL_BLUE, "Run dequeue on device %d",
+			mp_params->deq_param.dev_id);
+	MP_APP_LOG(INFO, COL_BLUE, "Run dequeue on qp %d",
+			mp_params->deq_param.qp_id);
+	i = 0;
+	while (mp_params->deq_param.vector_number[i] > 0 &&
+			i < MP_APP_MAX_VECTORS)	{
+		MP_APP_LOG(INFO, COL_BLUE, "Run dequeue vector %d",
+				mp_params->deq_param.vector_number[i]);
+		i++;
+	}
+
+	mp_params->deq_param.checkpoint = 1000000;
+
+	return 0;
+err:
+	MP_APP_LOG_2(ERR, COL_RED, "Error parsing deq");
+	return -1;
+}
+
+static int
+parse_print_stats(struct mp_crypto_app_parameters *mp_params,
+			const char *arg __rte_unused)
+{
+	mp_params->print_stats = 1;
+	return 0;
+}
+
+static int
+parse_session_mask(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	char *end = NULL;
+
+	mp_params->session_mask = strtoull(arg, &end, 16);
+
+	return 0;
+}
+
+static int
+parse_devtype(struct mp_crypto_app_parameters *mp_params,
+					const char *arg)
+{
+	if (arg == NULL) {
+		RTE_LOG(ERR, USER1, "--%s param argument is null\n",
+			MP_DEVTYPE_NAME);
+	}
+
+	if (strlen(arg) > (sizeof(mp_params->devtype_name) - 1)) {
+		RTE_LOG(ERR, USER1, "--%s different lengths\n",
+			MP_DEVTYPE_NAME);
+		return 0;
+	}
+
+	strlcpy(mp_params->devtype_name, arg,
+			sizeof(mp_params->devtype_name));
+
+	return 0;
+};
+
+typedef int (*option_parser_t)(struct mp_crypto_app_parameters
+			*mp_params,	const char *arg);
+
+struct long_opt_parser {
+	const char *lgopt_name;
+	option_parser_t parser_fn;
+};
+
+static int
+opts_parse_long(int opt_idx, struct mp_crypto_app_parameters *mp_params)
+{
+	struct long_opt_parser parsermap[] = {
+		{ MP_DEV_CONFIGURE, parse_config_dev },
+		{ MP_QP_CONFIGURE, parse_qp_config },
+		{ MP_ENQ, parse_enq },
+		{ MP_DEQ, parse_deq },
+		{ MP_PRINT_STATS, parse_print_stats },
+		{ MP_SESSION_MASK, parse_session_mask },
+		{ MP_DEVTYPE_NAME, parse_devtype },
+	};
+	unsigned int i;
+
+	for (i = 0; i < RTE_DIM(parsermap); i++) {
+		if (strncmp(lgopts[opt_idx].name, parsermap[i].lgopt_name,
+				strlen(lgopts[opt_idx].name)) == 0) {
+			return parsermap[i].parser_fn(mp_params, optarg);
+		}
+	}
+
+	return 0;
+}
+
+int
+options_parse(struct mp_crypto_app_parameters *mp_params,
+					int argc, char **argv)
+{
+	int opt, retval;
+	int opt_idx;
+
+	while ((opt = getopt_long(argc, argv, "h", lgopts, &opt_idx))
+			!= EOF) {
+		switch (opt) {
+		case 'h':
+			usage(argv[0]);
+			rte_exit(0, "Select options as above.\n");
+			break;
+		case 0:
+			retval = opts_parse_long(opt_idx, mp_params);
+			if (retval != 0)
+				return retval;
+			break;
+		default:
+			RTE_LOG(ERR, USER1, "Parse error after %s\n",
+					lgopts[opt_idx].name);
+			usage(argv[0]);
+			return 0;
+		}
+	}
+
+	return 0;
+}
+
+void
+options_default(struct mp_crypto_app_parameters *mp_params)
+{
+	int i = 0;
+
+	for (i = 0; i < MP_APP_MAX_VECTORS; i++) {
+		mp_params->enq_param.dev_id = -1;
+		mp_params->enq_param.qp_id = -1;
+		mp_params->enq_param.vector_number[i] = -1;
+		mp_params->deq_param.dev_id = -1;
+		mp_params->deq_param.qp_id = -1;
+		mp_params->deq_param.vector_number[i] = -1;
+	}
+
+	mp_params->enq_param.ops_no = 0;
+	mp_params->deq_param.ops_no = 0;
+	mp_params->print_stats = 0;
+}
+
+int
+options_check(__rte_unused struct mp_crypto_app_parameters *mp_params)
+{
+	return 0;
+}
diff --git a/examples/mp_crypto/mp_crypto_parser.h b/examples/mp_crypto/mp_crypto_parser.h
new file mode 100644
index 0000000..dfbde82
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_parser.h
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#ifndef _MP_CRYPTO_SAMPLE_APP_PARSER_
+#define _MP_CRYPTO_SAMPLE_APP_PARSER_
+
+#include <rte_hexdump.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev.h>
+#include <cmdline_parse.h>
+
+/* Make debug colorful! */
+#define COL_NORM	"\x1B[0m"
+#define COL_WHITE	"\x1B[37m"
+#define COL_RED		"\x1B[31m"
+#define COL_GREEN	"\x1B[32m"
+#define COL_YEL		"\x1B[33m"
+#define COL_BLUE	"\x1B[34m"
+#define COL_MAG		"\x1B[35m"
+
+#define MP_APP_LOG(level, color, str, args...) \
+	do {		\
+	printf("%s", color);			\
+	RTE_LOG(level, USER1, str, args);	\
+	printf("%s\n", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_LOG_2(level, color, str) \
+	do {		\
+	printf("%s", color);			\
+	RTE_LOG(level, USER1, str);	\
+	printf("%s\n", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_LOG_NO_RET(level, color, str, args...) \
+	do {		\
+	printf("\r%s", color);			\
+	RTE_LOG(level, USER1, str, args);	\
+	printf("%s", COL_NORM);	\
+	} while (0)
+
+#define MP_APP_QP_PARAM_LEN		(64 * 4)
+#define MP_APP_ENQ_PARAM_LEN	1024
+
+#define EMPTY_FLAGS		0
+
+#define MP_DEVTYPE_NAME		("devtype")
+#define MP_DEV_CONFIGURE	("config-dev")
+#define MP_QP_CONFIGURE		("qp-config")
+#define MP_ENQ				("enq")
+#define MP_DEQ				("deq")
+#define MP_SESSION_MASK		("session-mask")
+#define MP_PRINT_STATS		("print-stats")
+
+#define MP_APP_MAX_VECTORS	64
+
+extern const char *comp_perf_test_type_strs[];
+/* Command line parameters */
+extern struct mp_crypto_app_parameters *mp_app_params;
+/* Parser params */
+
+static const char livesign_print_char[4] = { '-', '\\', '|', '/'};
+
+int16_t
+get_options(int argc, char *argv[]);
+
+struct mp_crypto_app_enqdeq {
+	int dev_id;
+	int qp_id;
+	int vector_number[MP_APP_MAX_VECTORS];
+	int ops_no;
+	int checkpoint;
+};
+
+#define QP_TO_CONFIGURE		(-2)
+
+struct mp_crypto_app_parameters {
+	char devtype_name[RTE_DEV_NAME_MAX_LEN];
+	/* Driver to be used in this process */
+	char qp_config[MP_APP_QP_PARAM_LEN];
+	/* Queue Pairs configuration per device in process
+	 * in format q0,q1;q0,q1;, '-' means queue pair will not
+	 * be configured
+	 * Example: queue_pairs="0,1;0,-;-,1;" means that
+	 * device 0 will configure queue pairs 0 and 1,
+	 * device 1 will configure queue pairs 0
+	 * device 2 will configure queue pairs 1
+	 * Devices are order dependent
+	 */
+	char flow_config[MP_APP_ENQ_PARAM_LEN];
+	/* Enqueue configuration per process
+	 * Format "[dev_id]=qp_id:[op,]
+	 * Example: [0]=0:[enq, deq];[1]=0:[enq]
+	 * Mean that for this process qp 0 on device 0 will be
+	 * enqueuing and dequeuing in one queue pair,
+	 * meanwhile device 0 will only enqueue data on qpair 0.
+	 * Other process can then dequeue this data with
+	 * [1]=0:[deq]
+	 */
+	uint64_t dev_to_configure_mask;
+	/* Devices to configure, uint64 bitmask
+	 * 1 means dev 0, 2 dev 1, 4 dev... etc
+	 */
+	uint64_t session_mask;
+	/* Session to be created by this process,
+	 * if session was already created this step will be ommited.
+	 * Usage: session-mask=0x6 -> create session number 1 and 2.
+	 * Number of session refer to predefined array of sessions
+	 */
+	char enq[MP_APP_ENQ_PARAM_LEN];
+	struct mp_crypto_app_enqdeq enq_param;
+	char deq[MP_APP_ENQ_PARAM_LEN];
+	struct mp_crypto_app_enqdeq deq_param;
+	/* Enqueue/dequeue string used by this process.
+	 * Usage: [dev_id]:[qp_id]:[crypto_vector],[crypto_vector]...
+	 * Example 2:1:0,1,2, -> device no 2 on qp 1 enqueues ops from
+	 * vectors 0, 1, 2 .note ',' comma needs to be put after last arg
+	 */
+	int print_stats;
+	/* Print stats on the end on flow function */
+
+	uint16_t qp_id;
+	uint16_t waiting_qp_id;
+
+	int16_t configure_device;
+	int16_t setup_qp;
+	int16_t create_session_pool;
+	int16_t create_op_pool;
+	int16_t init_sessions;
+	int16_t build_ops;
+	int16_t dequeue;
+	int16_t enqueue;
+	int16_t dump_mempools;
+};
+
+int
+options_parse(struct mp_crypto_app_parameters *mp_params, int argc,
+			char **argv);
+void
+options_default(struct mp_crypto_app_parameters *mp_params);
+
+int
+options_check(struct mp_crypto_app_parameters *mp_params);
+
+#endif
diff --git a/examples/mp_crypto/mp_crypto_vectors.c b/examples/mp_crypto/mp_crypto_vectors.c
new file mode 100644
index 0000000..3a6b057
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_vectors.c
@@ -0,0 +1,175 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#include "mp_crypto_vectors.h"
+
+const struct mp_crypto_session_vector session_vectors[] = {
+	{ /* Session 0 */
+		.aead_op = RTE_CRYPTO_AEAD_OP_ENCRYPT,
+		.x_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+		.aead_algo = RTE_CRYPTO_AEAD_AES_GCM,
+		.crypto_key = {
+			.data = {
+				0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c,
+				0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08
+			},
+			.len = 16,
+		},
+		.aad_len = 0,
+		.iv_len = 12,
+		.digest_len = 16,
+	},
+	{ /* Session 1 */
+		.aead_op = RTE_CRYPTO_AEAD_OP_ENCRYPT,
+		.x_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+		.aead_algo = RTE_CRYPTO_AEAD_AES_GCM,
+		.crypto_key = {
+			.data = {
+				0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C,
+				0x6D, 0x6A, 0x8F, 0x94, 0x67, 0x30, 0x83, 0x08,
+				0xFE, 0xFF, 0xE9, 0x92, 0x86, 0x65, 0x73, 0x1C
+			},
+			.len = 24,
+		},
+		.aad_len = 0,
+		.iv_len = 12,
+		.digest_len = 16,
+	},
+};
+const uint64_t mp_app_numof_sessions =
+				RTE_DIM(session_vectors);
+/* Number of all sessions in array */
+
+const struct mp_crypto_vector vectors[] = {
+		{ /* Vector 0 AES128-GCM */
+			.session = 0,
+			.plaintext = {
+				.data = {
+					0xd9, 0x31, 0x32, 0x25,
+					0xf8, 0x84, 0x06, 0xe5,
+					0xa5, 0x59, 0x09, 0xc5,
+					0xaf, 0xf5, 0x26, 0x9a,
+					0x86, 0xa7, 0xa9, 0x53,
+					0x15, 0x34, 0xf7, 0xda,
+					0x2e, 0x4c, 0x30, 0x3d,
+					0x8a, 0x31, 0x8a, 0x72,
+					0x1c, 0x3c, 0x0c, 0x95,
+					0x95, 0x68, 0x09, 0x53,
+					0x2f, 0xcf, 0x0e, 0x24,
+					0x49, 0xa6, 0xb5, 0x25,
+					0xb1, 0x6a, 0xed, 0xf5,
+					0xaa, 0x0d, 0xe6, 0x57,
+					0xba, 0x63, 0x7b, 0x39,
+				},
+				.len = 60,
+			},
+			.iv = {
+					0xca, 0xfe, 0xba, 0xbe,
+					0xfa, 0xce, 0xdb, 0xad,
+					0xde, 0xca, 0xf8, 0x88
+			},
+			.ciphertext = {
+				.data = {
+					0x42, 0x83, 0x1e, 0xc2,
+					0x21, 0x77, 0x74, 0x24,
+					0x4b, 0x72, 0x21, 0xb7,
+					0x84, 0xd0, 0xd4, 0x9c,
+					0xe3, 0xaa, 0x21, 0x2f,
+					0x2c, 0x02, 0xa4, 0xe0,
+					0x35, 0xc1, 0x7e, 0x23,
+					0x29, 0xac, 0xa1, 0x2e,
+					0x21, 0xd5, 0x14, 0xb2,
+					0x54, 0x66, 0x93, 0x1c,
+					0x7d, 0x8f, 0x6a, 0x5a,
+					0xac, 0x84, 0xaa, 0x05,
+					0x1b, 0xa3, 0x0b, 0x39,
+					0x6a, 0x0a, 0xac, 0x97,
+					0x3d, 0x58, 0xe0, 0x91
+				},
+				.len = 60
+			},
+			.aad = {
+				.data = {
+
+				},
+			},
+			.digest = {
+				.data = {
+					0xA2, 0xA4, 0x35, 0x75,
+					0xDC, 0xB0, 0x57, 0x74,
+					0x07, 0x02, 0x30, 0xC2,
+					0xE7, 0x52, 0x02, 0x00},
+				.len = 16
+			},
+		},
+		{ /* Vector 1 AES192-GCM */
+			.session = 1,
+			.plaintext = {
+				.data = {
+					0xD9, 0x31, 0x32, 0x25,
+					0xF8, 0x84, 0x06, 0xE5,
+					0xA5, 0x59, 0x09, 0xC5,
+					0xAF, 0xF5, 0x26, 0x9A,
+					0x86, 0xA7, 0xA9, 0x53,
+					0x15, 0x34, 0xF7, 0xDA,
+					0x2E, 0x4C, 0x30, 0x3D,
+					0x8A, 0x31, 0x8A, 0x72,
+					0x1C, 0x3C, 0x0C, 0x95,
+					0x95, 0x68, 0x09, 0x53,
+					0x2F, 0xCF, 0x0E, 0x24,
+					0x49, 0xA6, 0xB5, 0x25,
+					0xB1, 0x6A, 0xED, 0xF5,
+					0xAA, 0x0D, 0xE6, 0x57,
+					0xBA, 0x63, 0x7B, 0x39,
+					0x1A, 0xAF, 0xD2, 0x55
+				},
+				.len = 60,
+			},
+			.iv = {
+				0xCA, 0xFE, 0xBA, 0xBE,
+				0xFA, 0xCE, 0xDB, 0xAD,
+				0xDE, 0xCA, 0xF8, 0x88
+			},
+			.ciphertext = {
+				.data = {
+					0x39, 0x80, 0xCA, 0x0B,
+					0x3C, 0x00, 0xE8, 0x41,
+					0xEB, 0x06, 0xFA, 0xC4,
+					0x87, 0x2A, 0x27, 0x57,
+					0x85, 0x9E, 0x1C, 0xEA,
+					0xA6, 0xEF, 0xD9, 0x84,
+					0x62, 0x85, 0x93, 0xB4,
+					0x0C, 0xA1, 0xE1, 0x9C,
+					0x7D, 0x77, 0x3D, 0x00,
+					0xC1, 0x44, 0xC5, 0x25,
+					0xAC, 0x61, 0x9D, 0x18,
+					0xC8, 0x4A, 0x3F, 0x47,
+					0x18, 0xE2, 0x44, 0x8B,
+					0x2F, 0xE3, 0x24, 0xD9,
+					0xCC, 0xDA, 0x27, 0x10,
+					0xAC, 0xAD, 0xE2, 0x56
+				},
+				.len = 60
+			},
+			.aad = {
+				.data = {
+
+				},
+			},
+			.digest = {
+				.data = {
+					0x99, 0x24, 0xA7, 0xC8,
+					0x58, 0x73, 0x36, 0xBF,
+					0xB1, 0x18, 0x02, 0x4D,
+					0xB8, 0x67, 0x4A, 0x14
+				},
+				.len = 16
+			},
+		},
+
+
+};
+
+const uint64_t mp_app_numof_ops =
+					RTE_DIM(vectors);
+/* Number of all operation instances */
diff --git a/examples/mp_crypto/mp_crypto_vectors.h b/examples/mp_crypto/mp_crypto_vectors.h
new file mode 100644
index 0000000..65386ad
--- /dev/null
+++ b/examples/mp_crypto/mp_crypto_vectors.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+#ifndef _MP_CRYPTO_SAMPLE_APP_VECTORS_
+#define _MP_CRYPTO_SAMPLE_APP_VECTORS_
+
+#include <rte_cryptodev.h>
+
+struct mp_crypto_session_vector {
+	union {
+		enum rte_crypto_aead_algorithm aead_algo;
+		enum rte_crypto_auth_algorithm auth_algo;
+		enum rte_crypto_cipher_algorithm cipher_algo;
+	};
+	enum rte_crypto_sym_xform_type x_type;
+	union {
+		enum rte_crypto_aead_operation aead_op;
+		enum rte_crypto_cipher_operation cipher_op;
+		enum rte_crypto_auth_operation auth_op;
+	};
+	struct {
+		uint8_t data[64];
+		uint16_t len;
+	} crypto_key;
+	struct {
+		uint8_t data[64];
+		uint16_t len;
+	} auth_key;
+	uint16_t aad_len;
+	uint16_t iv_len;
+	uint16_t digest_len;
+	int chained;
+};
+
+struct mp_crypto_vector {
+	int session;
+	struct {
+		uint8_t data[2048];
+		int len;
+	} ciphertext;
+	struct {
+		uint8_t data[2048];
+		int len;
+	} plaintext;
+	struct {
+		uint8_t data[2048];
+		int len;
+	} digest;
+	struct {
+		uint8_t data[64];
+	} aad;
+	uint8_t iv[16];
+};
+
+/* Predefinced vectors */
+extern const struct mp_crypto_session_vector session_vectors[];
+/* Sessions vectors for this device */
+extern const uint64_t mp_app_numof_sessions;
+/* Number of all sessions in array */
+
+extern const struct mp_crypto_vector vectors[];
+/* Operation vectors for this device */
+const uint64_t mp_app_numof_ops;
+/* Total number of operation types */
+
+#endif
-- 
2.1.0